From 919f64d32d3980b21ad3e7174f37a7062249408b Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Mon, 11 Mar 2024 15:09:07 +0000 Subject: [PATCH] disable kube-proxy, address review comments --- controller/linodemachine_controller.go | 12 +++- .../linodemachine_controller_helpers.go | 57 ++++++++++++++++--- templates/addons/cilium/cilium.yaml | 10 ++++ .../addons/provider-linode/linode-ccm.yaml | 5 ++ templates/common-init-files/secret.yaml | 1 + templates/flavors/base/linodeCluster.yaml | 2 +- .../flavors/base/linodeMachineTemplate.yaml | 4 +- templates/flavors/base/linodeVPC.yaml | 4 +- .../flavors/default/kubeadmControlPlane.yaml | 2 + templates/flavors/k3s/k3sConfigTemplate.yaml | 2 +- templates/flavors/k3s/k3sControlPlane.yaml | 32 ++++++++++- templates/flavors/rke2/kustomization.yaml | 12 ++++ .../flavors/rke2/rke2ConfigTemplate.yaml | 4 +- templates/flavors/rke2/rke2ControlPlane.yaml | 29 +--------- 14 files changed, 131 insertions(+), 45 deletions(-) diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index 74fa36b6d..0f8f91ae8 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -70,6 +70,11 @@ var requeueInstanceStatuses = map[linodego.InstanceStatus]bool{ linodego.InstanceResizing: true, } +type nodeIP struct { + ip string + ipType clusterv1.MachineAddressType +} + // LinodeMachineReconciler reconciles a LinodeMachine object type LinodeMachineReconciler struct { client.Client @@ -295,7 +300,12 @@ func (r *LinodeMachineReconciler) reconcileCreate( machineScope.LinodeMachine.Status.Ready = true machineScope.LinodeMachine.Spec.InstanceID = &linodeInstance.ID machineScope.LinodeMachine.Spec.ProviderID = util.Pointer(fmt.Sprintf("linode://%d", linodeInstance.ID)) - machineScope.LinodeMachine.Status.Addresses = buildInstanceAddrs(linodeInstance) + + addrs, err := r.buildInstanceAddrs(ctx, logger, machineScope, linodeInstance.ID) + if err != nil { + return linodeInstance, err + } + machineScope.LinodeMachine.Status.Addresses = addrs if err = services.AddNodeToNB(ctx, logger, machineScope); err != nil { logger.Error(err, "Failed to add instance to Node Balancer backend") diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index c4c6afac1..24cfe5557 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -115,20 +115,59 @@ func (r *LinodeMachineReconciler) newCreateConfig(ctx context.Context, machineSc return createConfig, nil } -func buildInstanceAddrs(linodeInstance *linodego.Instance) []clusterv1.MachineAddress { +func (r *LinodeMachineReconciler) buildInstanceAddrs(ctx context.Context, logger logr.Logger, machineScope *scope.MachineScope, instanceID int) ([]clusterv1.MachineAddress, error) { addrs := []clusterv1.MachineAddress{} - for _, addr := range linodeInstance.IPv4 { - addrType := clusterv1.MachineExternalIP - if addr.IsPrivate() { - addrType = clusterv1.MachineInternalIP - } + ips, err := r.getInstanceIPv4Addresses(ctx, logger, machineScope, instanceID) + if err != nil { + logger.Error(err, "Failed to get instance ip addresses") + return nil, err + } + + // add all instance ips to machine's status + for _, ip := range ips { addrs = append(addrs, clusterv1.MachineAddress{ - Type: addrType, - Address: addr.String(), + Type: ip.ipType, + Address: ip.ip, }) } - return addrs + return addrs, nil +} + +func (r *LinodeMachineReconciler) getInstanceIPv4Addresses(ctx context.Context, logger logr.Logger, machineScope *scope.MachineScope, instanceID int) ([]nodeIP, error) { + addresses, err := machineScope.LinodeClient.GetInstanceIPAddresses(ctx, instanceID) + if err != nil { + return nil, err + } + + // get the default instance config + configs, err := machineScope.LinodeClient.ListInstanceConfigs(ctx, instanceID, &linodego.ListOptions{}) + if err != nil || len(configs) == 0 { + logger.Error(err, "Failed to list instance configs") + return nil, err + } + + ips := []nodeIP{} + // check if a node has public ip and store it + if len(addresses.IPv4.Public) != 0 { + ips = append(ips, nodeIP{ip: addresses.IPv4.Public[0].Address, ipType: clusterv1.MachineExternalIP}) + } + + // Iterate over interfaces in config and find VPC specific ips + for _, iface := range configs[0].Interfaces { + if iface.VPCID != nil && iface.IPv4.VPC != "" { + ips = append(ips, nodeIP{ip: iface.IPv4.VPC, ipType: clusterv1.MachineInternalIP}) + } + } + + // if a node has private ip, store it as well + // NOTE: We specifically store VPC ips first so that they are used first during + // bootstrap when we set `registrationMethod: internal-only-ips` + if len(addresses.IPv4.Private) != 0 { + ips = append(ips, nodeIP{ip: addresses.IPv4.Private[0].Address, ipType: clusterv1.MachineInternalIP}) + } + + return ips, nil } func (r *LinodeMachineReconciler) getOwnerMachine(ctx context.Context, linodeMachine infrav1alpha1.LinodeMachine, log logr.Logger) (*clusterv1.Machine, error) { diff --git a/templates/addons/cilium/cilium.yaml b/templates/addons/cilium/cilium.yaml index 05e33ba9b..9fa76e721 100644 --- a/templates/addons/cilium/cilium.yaml +++ b/templates/addons/cilium/cilium.yaml @@ -20,6 +20,12 @@ spec: ipv4NativeRoutingCIDR: 10.0.0.0/8 tunnelProtocol: "" enableIPv4Masquerade: true + egressMasqueradeInterfaces: eth0 + k8sServiceHost: {{ .InfraCluster.spec.controlPlaneEndpoint.host }} + k8sServicePort: {{ .InfraCluster.spec.controlPlaneEndpoint.port }} + extraArgs: + - --direct-routing-device=eth1 + - --nodeport-addresses=0.0.0.0/0 ipam: mode: kubernetes ipv4: @@ -33,3 +39,7 @@ spec: enabled: true ui: enabled: true +# ipMasqAgent: +# enabled: true +# bpf: +# masquerade: true diff --git a/templates/addons/provider-linode/linode-ccm.yaml b/templates/addons/provider-linode/linode-ccm.yaml index dd2839fa4..5e4715390 100644 --- a/templates/addons/provider-linode/linode-ccm.yaml +++ b/templates/addons/provider-linode/linode-ccm.yaml @@ -15,6 +15,11 @@ spec: wait: true timeout: 5m valuesTemplate: | + routeController: + vpcName: ${VPC_NAME:=${CLUSTER_NAME}} + linodeNodePrivateSubnet: 10.0.0.0/8 + configureCloudRoutes: true + routeReconciliationPeriod: 1m secretRef: name: "linode-token-region" image: diff --git a/templates/common-init-files/secret.yaml b/templates/common-init-files/secret.yaml index e2b3a7827..1160f367d 100644 --- a/templates/common-init-files/secret.yaml +++ b/templates/common-init-files/secret.yaml @@ -25,6 +25,7 @@ stringData: set -euo pipefail export DEBIAN_FRONTEND=noninteractive hostnamectl set-hostname "$1" && hostname -F /etc/hostname + echo "$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1) $1" >> /etc/hosts mkdir -p -m 755 /etc/apt/keyrings PATCH_VERSION=$${2#[v]} VERSION=$${PATCH_VERSION%.*} diff --git a/templates/flavors/base/linodeCluster.yaml b/templates/flavors/base/linodeCluster.yaml index d4cb85701..565a89ff3 100644 --- a/templates/flavors/base/linodeCluster.yaml +++ b/templates/flavors/base/linodeCluster.yaml @@ -10,4 +10,4 @@ spec: vpcRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: LinodeVPC - name: vpc-${CLUSTER_NAME} + name: ${VPC_NAME:=${CLUSTER_NAME}} diff --git a/templates/flavors/base/linodeMachineTemplate.yaml b/templates/flavors/base/linodeMachineTemplate.yaml index b2ee8f19d..81731faca 100644 --- a/templates/flavors/base/linodeMachineTemplate.yaml +++ b/templates/flavors/base/linodeMachineTemplate.yaml @@ -11,7 +11,7 @@ spec: region: ${LINODE_REGION} authorizedKeys: # uncomment to include your ssh key in linode provisioning - # - ${LINODE_SSH_PUBKEY:=""} + - ${LINODE_SSH_PUBKEY:=""} --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: LinodeMachineTemplate @@ -25,4 +25,4 @@ spec: region: ${LINODE_REGION} authorizedKeys: # uncomment to include your ssh key in linode provisioning - # - ${LINODE_SSH_PUBKEY:=""} + - ${LINODE_SSH_PUBKEY:=""} diff --git a/templates/flavors/base/linodeVPC.yaml b/templates/flavors/base/linodeVPC.yaml index 515e5a176..4ed39878c 100644 --- a/templates/flavors/base/linodeVPC.yaml +++ b/templates/flavors/base/linodeVPC.yaml @@ -2,9 +2,9 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: LinodeVPC metadata: - name: vpc-${CLUSTER_NAME} + name: ${VPC_NAME:=${CLUSTER_NAME}} spec: region: ${LINODE_REGION} subnets: - ipv4: 10.0.0.0/8 - label: default \ No newline at end of file + label: default diff --git a/templates/flavors/default/kubeadmControlPlane.yaml b/templates/flavors/default/kubeadmControlPlane.yaml index 18da0769a..00c5dc6c2 100644 --- a/templates/flavors/default/kubeadmControlPlane.yaml +++ b/templates/flavors/default/kubeadmControlPlane.yaml @@ -56,6 +56,8 @@ spec: - - LABEL=etcd_data - /var/lib/etcd_data initConfiguration: + skipPhases: + - addon/kube-proxy nodeRegistration: kubeletExtraArgs: cloud-provider: external diff --git a/templates/flavors/k3s/k3sConfigTemplate.yaml b/templates/flavors/k3s/k3sConfigTemplate.yaml index 9127f4eab..cf551919d 100644 --- a/templates/flavors/k3s/k3sConfigTemplate.yaml +++ b/templates/flavors/k3s/k3sConfigTemplate.yaml @@ -13,6 +13,6 @@ spec: preK3sCommands: - | mkdir -p /etc/rancher/k3s/config.yaml.d/ - echo "node-ip: $(hostname -I | grep -oE 192\.168\.[0-9]+\.[0-9]+)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml + echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a diff --git a/templates/flavors/k3s/k3sControlPlane.yaml b/templates/flavors/k3s/k3sControlPlane.yaml index 0b19633b3..bafb1ea05 100644 --- a/templates/flavors/k3s/k3sControlPlane.yaml +++ b/templates/flavors/k3s/k3sControlPlane.yaml @@ -33,6 +33,36 @@ spec: name: linode-${CLUSTER_NAME}-crs-0 owner: root:root path: /var/lib/rancher/k3s/server/manifests/linode-token-region.yaml + - path: /var/lib/rancher/k3s/server/manifests/k3s-cilium-config.yaml + owner: root:root + permissions: "0640" + content: | + apiVersion: helm.cattle.io/v1 + kind: HelmChartConfig + metadata: + name: cilium + namespace: kube-system + spec: + valuesContent: |- + routingMode: native + kubeProxyReplacement: true + ipv4NativeRoutingCIDR: 10.0.0.0/8 + tunnelProtocol: "" + enableIPv4Masquerade: true + egressMasqueradeInterfaces: eth0 + k8sServiceHost: 10.0.0.2 + k8sServicePort: 6443 + extraArgs: + - --direct-routing-device=eth1 + - --nodeport-addresses=0.0.0.0/0 + ipam: + mode: kubernetes + ipv4: + enabled: true + ipv6: + enabled: false + k8s: + requireIPv4PodCIDR: true serverConfig: disableComponents: - servicelb @@ -43,7 +73,7 @@ spec: - "provider-id=linode://{{ ds.meta_data.id }}" preK3sCommands: - | - echo "node-ip: $(hostname -I | grep -oE 192\.168\.[0-9]+\.[0-9]+)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml + echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/k3s/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a replicas: ${CONTROL_PLANE_MACHINE_COUNT} diff --git a/templates/flavors/rke2/kustomization.yaml b/templates/flavors/rke2/kustomization.yaml index 34108ba7f..2c5df66f7 100644 --- a/templates/flavors/rke2/kustomization.yaml +++ b/templates/flavors/rke2/kustomization.yaml @@ -5,6 +5,7 @@ resources: - rke2ControlPlane.yaml - rke2ConfigTemplate.yaml - secret.yaml + - ../../addons/cilium patches: - target: group: cluster.x-k8s.io @@ -14,6 +15,17 @@ patches: - op: replace path: /spec/controlPlaneRef/kind value: RKE2ControlPlane + - target: + group: cluster.x-k8s.io + version: v1beta1 + kind: Cluster + patch: |- + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Cluster + metadata: + name: ${CLUSTER_NAME} + labels: + cni: cilium - target: group: cluster.x-k8s.io version: v1beta1 diff --git a/templates/flavors/rke2/rke2ConfigTemplate.yaml b/templates/flavors/rke2/rke2ConfigTemplate.yaml index 8d2a0feff..334f84511 100644 --- a/templates/flavors/rke2/rke2ConfigTemplate.yaml +++ b/templates/flavors/rke2/rke2ConfigTemplate.yaml @@ -12,11 +12,11 @@ spec: kubelet: extraArgs: - "provider-id=linode://{{ ds.meta_data.id }}" - # TODO: use MDS to get public and private IP instead because hostname ordering can't always be assumed + # TODO: use MDS to get private IP instead preRKE2Commands: - | mkdir -p /etc/rancher/rke2/config.yaml.d/ - echo "node-ip: $(hostname -I | grep -oE 192\.168\.[0-9]+\.[0-9]+)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml + echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname diff --git a/templates/flavors/rke2/rke2ControlPlane.yaml b/templates/flavors/rke2/rke2ControlPlane.yaml index 4360827f7..e18ae9e47 100644 --- a/templates/flavors/rke2/rke2ControlPlane.yaml +++ b/templates/flavors/rke2/rke2ControlPlane.yaml @@ -21,39 +21,16 @@ spec: name: linode-${CLUSTER_NAME}-crs-0 owner: root:root path: /var/lib/rancher/rke2/server/manifests/linode-token-region.yaml - - path: /var/lib/rancher/rke2/server/manifests/rke2-cilium-config.yaml - owner: root:root - permissions: "0640" - content: | - apiVersion: helm.cattle.io/v1 - kind: HelmChartConfig - metadata: - name: rke2-cilium - namespace: kube-system - spec: - valuesContent: |- - routingMode: native - kubeProxyReplacement: true - ipv4NativeRoutingCIDR: 10.0.0.0/8 - tunnelProtocol: "" - enableIPv4Masquerade: true - ipam: - mode: kubernetes - ipv4: - enabled: true - ipv6: - enabled: false - k8s: - requireIPv4PodCIDR: true registrationMethod: internal-only-ips serverConfig: - cni: cilium + cni: none cloudProviderName: external disableComponents: pluginComponents: - "rke2-ingress-nginx" kubernetesComponents: - "cloudController" + - "kubeProxy" agentConfig: version: ${RKE2_KUBERNETES_VERSION} nodeName: '{{ ds.meta_data.label }}' @@ -63,7 +40,7 @@ spec: preRKE2Commands: - | mkdir -p /etc/rancher/rke2/config.yaml.d/ - echo "node-ip: $(hostname -I | grep -oE 192\.168\.[0-9]+\.[0-9]+)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml + echo "node-ip: $(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)" >> /etc/rancher/rke2/config.yaml.d/capi-config.yaml - sed -i '/swap/d' /etc/fstab - swapoff -a - hostnamectl set-hostname '{{ ds.meta_data.label }}' && hostname -F /etc/hostname