Merge pull request #698 from l1b0k/policy

Policy: add per package lb control

policy/cilium/0032-fix-disable-per-package-lb.patch

@@ -0,0 +1,101 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: l1b0k <[email protected]>
+Date: Sat, 12 Oct 2024 09:33:24 +0800
+Subject: fix disable per-package lb
+
+Signed-off-by: l1b0k <[email protected]>
+---
+ bpf/bpf_lxc.c                       | 5 ++---
+ daemon/cmd/daemon_main.go           | 3 +++
+ pkg/datapath/linux/config/config.go | 5 +++++
+ pkg/option/config.go                | 6 ++++++
+ 4 files changed, 16 insertions(+), 3 deletions(-)
+
+diff --git a/bpf/bpf_lxc.c b/bpf/bpf_lxc.c
+index 2773e2c276..7e424204e2 100644
+--- a/bpf/bpf_lxc.c
++++ b/bpf/bpf_lxc.c
+@@ -58,9 +58,9 @@
+  * Most services with L7 LB flag can not be redirected to their proxy port
+  * in bpf_sock, so we must check for those via per packet LB as well.
+  */
+-#if !defined(ENABLE_SOCKET_LB_FULL) || \
++#if (!defined(ENABLE_SOCKET_LB_FULL) || \
+     defined(ENABLE_SOCKET_LB_HOST_ONLY) || \
+-    defined(ENABLE_L7_LB)
++    defined(ENABLE_L7_LB) ) && !defined(DISABLE_PER_PACKET_LB)
+ # define ENABLE_PER_PACKET_LB 1
+ #endif
+
+@@ -1247,7 +1247,6 @@ static __always_inline int __tail_handle_ipv4(struct __ctx_buff *ctx)
+
+ 	if (unlikely(!is_valid_lxc_src_ipv4(ip4)))
+ 		return DROP_INVALID_SIP;
+-
+ #ifdef ENABLE_PER_PACKET_LB
+ 	{
+ 		struct ipv4_ct_tuple tuple = {};
+diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go
+index de2ad4900b..8b495d972f 100644
+--- a/daemon/cmd/daemon_main.go
++++ b/daemon/cmd/daemon_main.go
+@@ -1147,6 +1147,9 @@ func initializeFlags() {
+ 	flags.MarkHidden(option.EnableStaleCiliumEndpointCleanup)
+ 	option.BindEnv(option.EnableStaleCiliumEndpointCleanup)
+
++	flags.Bool(option.DisablePerPacketLB, false, "Disable svc func.")
++	option.BindEnv(option.DisablePerPacketLB)
++
+ 	viper.BindPFlags(flags)
+ }
+
+diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go
+index 060328adf9..4201e878fb 100644
+--- a/pkg/datapath/linux/config/config.go
++++ b/pkg/datapath/linux/config/config.go
+@@ -98,6 +98,11 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC
+
+ 	cDefinesMap["KERNEL_HZ"] = fmt.Sprintf("%d", option.Config.KernelHz)
+
++	if option.Config.DisablePerPacketLB {
++		log.Infof("Disabling per-packet LB policy")
++		cDefinesMap["DISABLE_PER_PACKET_LB"] = "1"
++	}
++
+ 	if option.Config.EnableIPv6 {
+ 		extraMacrosMap["ROUTER_IP"] = routerIP.String()
+ 		fw.WriteString(defineIPv6("ROUTER_IP", routerIP))
+diff --git a/pkg/option/config.go b/pkg/option/config.go
+index d4d9643f4e..80b5eb2c7c 100644
+--- a/pkg/option/config.go
++++ b/pkg/option/config.go
+@@ -1104,6 +1104,8 @@ const (
+ 	// EnableStaleCiliumEndpointCleanup sets whether Cilium should perform cleanup of
+ 	// stale CiliumEndpoints during init.
+ 	EnableStaleCiliumEndpointCleanup = "enable-stale-cilium-endpoint-cleanup"
++
++	DisablePerPacketLB = "disable-per-package-lb"
+ )
+
+ // Default string arguments
+@@ -2262,6 +2264,8 @@ type DaemonConfig struct {
+ 	// This will attempt to remove local CiliumEndpoints that are not managed by Cilium
+ 	// following Endpoint restoration.
+ 	EnableStaleCiliumEndpointCleanup bool
++
++	DisablePerPacketLB bool
+ }
+
+ var (
+@@ -3257,6 +3261,8 @@ func (c *DaemonConfig) Populate() {
+ 	c.EnableK8sTerminatingEndpoint = viper.GetBool(EnableK8sTerminatingEndpoint)
+ 	c.EnableStaleCiliumEndpointCleanup = viper.GetBool(EnableStaleCiliumEndpointCleanup)
+
++	c.DisablePerPacketLB = viper.GetBool(DisablePerPacketLB)
++
+ 	// Disable Envoy version check if L7 proxy is disabled.
+ 	c.DisableEnvoyVersionCheck = viper.GetBool(DisableEnvoyVersionCheck)
+ 	if !c.EnableL7Proxy {
+-- 
+2.46.0
+

policy/policyinit.sh

@@ -5,19 +5,19 @@ mount -o remount rw /proc/sys
 export DATASTORE_TYPE=kubernetes
 
 masq_eni_only() {
-    if ! "$1" -t nat -L terway-masq; then
-        # Create a new chain in nat table.
-        "$1" -t nat -N terway-masq
-    fi
+  if ! "$1" -t nat -L terway-masq; then
+    # Create a new chain in nat table.
+    "$1" -t nat -N terway-masq
+  fi
 
-    if ! "$1" -t nat -L POSTROUTING | grep -q terway-masq; then
-        # Append that chain to POSTROUTING table.
-        "$1" -t nat -A POSTROUTING -m comment --comment "terway:masq-outgoing" ! -o lo -j terway-masq
-    fi
+  if ! "$1" -t nat -L POSTROUTING | grep -q terway-masq; then
+    # Append that chain to POSTROUTING table.
+    "$1" -t nat -A POSTROUTING -m comment --comment "terway:masq-outgoing" ! -o lo -j terway-masq
+  fi
 
-    if ! "$1" -t nat -L terway-masq | grep -q MASQUERADE; then
-       "$1" -t nat -A terway-masq -j MASQUERADE
-    fi
+  if ! "$1" -t nat -L terway-masq | grep -q MASQUERADE; then
+    "$1" -t nat -A terway-masq -j MASQUERADE
+  fi
 }
 
 terway_config_val() {
@@ -51,16 +51,16 @@ if grep -q "cni_exclusive_eni *= *eniOnly" "$node_capabilities"; then
 
   # for health check
   if [ "$FELIX_HEALTHPORT" != "" ]; then
-      # shellcheck disable=SC2016
-      exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
+    # shellcheck disable=SC2016
+    exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
   else
-      # shellcheck disable=SC2016
-      exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
+    # shellcheck disable=SC2016
+    exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
   fi
 fi
 
 if grep -q "datapath *= *datapathv2" "$node_capabilities"; then
-      datapath_mode=veth
+  datapath_mode=veth
 fi
 
 # kernel version has already checked in initContainer, so just determine whether plugin chaining exists
@@ -69,7 +69,7 @@ if [ "$virtyal_type" = "ipvlan" ] || [ "$virtyal_type" = "datapathv2" ]; then
 
   # kernel version equal and above 4.19
   if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } ||
-     [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
+    [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
 
     extra_args=$(terway_config_val 'cilium_args')
     if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
@@ -95,106 +95,115 @@ if [ "$virtyal_type" = "ipvlan" ] || [ "$virtyal_type" = "datapathv2" ]; then
       echo "turning up hubble, passing args \"${extra_args}\""
     fi
 
-		if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then
-				extra_args="${extra_args} --enable-in-cluster-loadbalance=true "
-				echo "turning up in cluster loadbalance, passing args \"${extra_args}\""
-		fi
+    if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then
+      extra_args="${extra_args} --enable-in-cluster-loadbalance=true "
+      echo "turning up in cluster loadbalance, passing args \"${extra_args}\""
+    fi
 
-		if bpftool -j feature probe | grep bpf_skb_ecn_set_ce ; then
-				extra_args="${extra_args} --enable-bandwidth-manager=true "
-		fi
+    if bpftool -j feature probe | grep bpf_skb_ecn_set_ce; then
+      extra_args="${extra_args} --enable-bandwidth-manager=true "
+    fi
 
     echo "using cilium as network routing & policy"
 
     # shellcheck disable=SC2086
     exec cilium-agent --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \
-         --enable-policy=$ENABLE_POLICY \
-         --agent-health-port=9099 --disable-envoy-version-check=true \
-         --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
-         --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
-         --install-iptables-rules=false --enable-l7-proxy=false \
-         --ipam=cluster-pool  --datapath-mode=${datapath_mode} --enable-runtime-device-detection=true ${extra_args}
+      --enable-policy=$ENABLE_POLICY \
+      --agent-health-port=9099 --disable-envoy-version-check=true \
+      --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
+      --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
+      --install-iptables-rules=false --enable-l7-proxy=false \
+      --ipam=cluster-pool --datapath-mode=${datapath_mode} --enable-runtime-device-detection=true ${extra_args}
   fi
 fi
-  # shellcheck disable=SC1091
-  source uninstall_policy.sh
-
-  # check kernel version
+# shellcheck disable=SC1091
+source uninstall_policy.sh
 
-  export FELIX_IPTABLESBACKEND=Auto
-  if ( uname -r | grep -E "el7|an7" && [ "${KERNEL_MAJOR_VERSION}" -eq 3 ] ) || ( uname -r | grep -E "al7" && [ "${KERNEL_MAJOR_VERSION}" -eq 4 ] ); then
-    export FELIX_IPTABLESBACKEND=Legacy
-  elif ( uname -r | grep -E "el8|an8" && [ "${KERNEL_MAJOR_VERSION}" -ge 4 ] ) || ( uname -r | grep -E "al8|lifsea8" && [ "${KERNEL_MAJOR_VERSION}" -ge 5 ] ); then
-    export FELIX_IPTABLESBACKEND=NFT
+# check kernel version
 
-    # clean legacy rules if exist
-    cleanup_legacy
-  fi
+export FELIX_IPTABLESBACKEND=Auto
+if (uname -r | grep -E "el7|an7" && [ "${KERNEL_MAJOR_VERSION}" -eq 3 ]) || (uname -r | grep -E "al7" && [ "${KERNEL_MAJOR_VERSION}" -eq 4 ]); then
+  export FELIX_IPTABLESBACKEND=Legacy
+elif (uname -r | grep -E "el8|an8" && [ "${KERNEL_MAJOR_VERSION}" -ge 4 ]) || (uname -r | grep -E "al8|lifsea8" && [ "${KERNEL_MAJOR_VERSION}" -ge 5 ]); then
+  export FELIX_IPTABLESBACKEND=NFT
 
-  # default for veth
-  export FELIX_LOGSEVERITYSYS=none
-  export FELIX_LOGSEVERITYSCREEN=info
-  export CALICO_NETWORKING_BACKEND=none
-  export CLUSTER_TYPE=k8s,aliyun
-  export CALICO_DISABLE_FILE_LOGGING=true
-  # shellcheck disable=SC2154
-  export CALICO_IPV4POOL_CIDR="${Network}"
-  export FELIX_IPTABLESREFRESHINTERVAL="${IPTABLESREFRESHINTERVAL:-60}"
-  export FELIX_IPV6SUPPORT=true
-  export WAIT_FOR_DATASTORE=true
-  export IP=""
-  export NO_DEFAULT_POOLS=true
-  export FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT
-  export FELIX_HEALTHENABLED=true
-  export FELIX_LOGFILEPATH=/dev/null
-  export FELIX_BPFENABLED=false
-  export FELIX_XDPENABLED=false
-  export FELIX_BPFCONNECTTIMELOADBALANCINGENABLED=false
-  export FELIX_BPFKUBEPROXYIPTABLESCLEANUPENABLED=false
-  exec 2>&1
-  if [ -n "$NODENAME" ]; then
-      export FELIX_FELIXHOSTNAME="$NODENAME"
-  fi
-  if [ -n "$DATASTORE_TYPE" ]; then
-      export FELIX_DATASTORETYPE="$DATASTORE_TYPE"
-  fi
+  # clean legacy rules if exist
+  cleanup_legacy
+fi
 
-  if [ "$network_policy_provider" = "ebpf" ]; then
-     cleanup_felix
-     # kernel version equal and above 4.19
-     if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } ||
-        [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
-
-       if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
-         ENABLE_POLICY="default"
-       else
-         ENABLE_POLICY="never"
-       fi
-       # shellcheck disable=SC2086
-       exec cilium-agent --kube-proxy-replacement=disabled --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \
-            --enable-policy=$ENABLE_POLICY \
-            --agent-health-port=9099 --disable-envoy-version-check=true \
-            --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
-            --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
-            --install-iptables-rules=false --enable-l7-proxy=false \
-            --ipam=cluster-pool
-     else
-       echo "unsupported kernel version"
-       exit 1
-     fi
-  else
-     if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
-        exec calico-felix
-     fi
-  fi
+# default for veth
+export FELIX_LOGSEVERITYSYS=none
+export FELIX_LOGSEVERITYSCREEN=info
+export CALICO_NETWORKING_BACKEND=none
+export CLUSTER_TYPE=k8s,aliyun
+export CALICO_DISABLE_FILE_LOGGING=true
+# shellcheck disable=SC2154
+export CALICO_IPV4POOL_CIDR="${Network}"
+export FELIX_IPTABLESREFRESHINTERVAL="${IPTABLESREFRESHINTERVAL:-60}"
+export FELIX_IPV6SUPPORT=true
+export WAIT_FOR_DATASTORE=true
+export IP=""
+export NO_DEFAULT_POOLS=true
+export FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT
+export FELIX_HEALTHENABLED=true
+export FELIX_LOGFILEPATH=/dev/null
+export FELIX_BPFENABLED=false
+export FELIX_XDPENABLED=false
+export FELIX_BPFCONNECTTIMELOADBALANCINGENABLED=false
+export FELIX_BPFKUBEPROXYIPTABLESCLEANUPENABLED=false
+exec 2>&1
+if [ -n "$NODENAME" ]; then
+  export FELIX_FELIXHOSTNAME="$NODENAME"
+fi
+if [ -n "$DATASTORE_TYPE" ]; then
+  export FELIX_DATASTORETYPE="$DATASTORE_TYPE"
+fi
 
-  config_masquerade
+if [ "$network_policy_provider" = "ebpf" ]; then
   cleanup_felix
-  # for health check
-  if [ "$FELIX_HEALTHPORT" != "" ]; then
-      # shellcheck disable=SC2016
-      exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
+  # kernel version equal and above 4.19
+  if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } ||
+    [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then
+
+    extra_args=$(terway_config_val 'cilium_args')
+
+    if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
+      ENABLE_POLICY="default"
+    else
+      ENABLE_POLICY="never"
+      extra_args="${extra_args} --labels=k8s:io\\.kubernetes\\.pod\\.namespace "
+    fi
+
+		if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then
+				extra_args="${extra_args} --enable-in-cluster-loadbalance=true "
+				echo "turning up in cluster loadbalance, passing args \"${extra_args}\""
+		fi
+
+    # shellcheck disable=SC2086
+    exec cilium-agent --kube-proxy-replacement=disabled --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \
+      --enable-policy=$ENABLE_POLICY \
+      --agent-health-port=9099 --disable-envoy-version-check=true \
+      --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \
+      --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \
+      --install-iptables-rules=false --enable-l7-proxy=false \
+      --ipam=cluster-pool ${extra_args}
   else
-      # shellcheck disable=SC2016
-      exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
+    echo "unsupported kernel version"
+    exit 1
+  fi
+else
+  if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then
+    exec calico-felix
   fi
+fi
+
+config_masquerade
+cleanup_felix
+# for health check
+if [ "$FELIX_HEALTHPORT" != "" ]; then
+  # shellcheck disable=SC2016
+  exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
+else
+  # shellcheck disable=SC2016
+  exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null'
+fi