Add ethtool link prop. Disable TX offload (unless --keep-veth-offload)

Add a new ethtool property that can be either a string or a string
array. The syntax of each ethtool property is basically the ethtool
command line without the "devname". So for the ethtool command:
    ethtool --offload eth0 rx off
the link equivalent would be:
    {dev: eth0, ethtool: ["--offload rx off"], ...}

Disable TX offload for veth container interfaces unless
`--keep-veth-offload` is enabled. Veth interfaces are a little weird in
that they have the TX offload setting enabled by default. However, this
is basically ignored (veth interfaces don't have hardware to offload
to). Usually this is fine and is the fastest setting. However, if you
have a some sort of software networking in the middle of veth links then
the kernel will lose track of the fact that it does not need to validate
checksums and having TX offload will cause the veth interface at the far
end to drop packets due to bad checksums (because they were never done).
The safer behavior is disable TX offload for veth links by default so
that the kernel will do software checksum generation even for veth
links. This changes a the default behavior and for most
testing/simulation situation should be a non-issue, however it could
some performance implications for high throughput scenarios. For this
reason the `--keep-veth-offload` parameter is provided for restoring the
previous default behavior.

If you want to keep the original behavior
but want to disable TX offload for certain veth interfaces you can use
`--keep-veth-offload` and then add `ethtool: "--offload tx off"` for
the specific interfaces that should have TX offload disabled.

Author: kanaka

README.md

@@ -146,6 +146,7 @@ The following table describes the link properties:
 | mode      | 5          | string         |         | virt intf mode           |
 | vlanid    | vlan       | number         |         | VLAN ID                  |
 | forward   | veth       | strings 6 8    |         | forward conlink ports 7  |
+| ethtool   | veth       | strings 8      |         | ethtool settings         |
 
 - 1 - veth, dummy, vlan, ipvlan, macvlan, ipvtap, macvtap
 - 2 - defaults to outer compose service
@@ -188,6 +189,12 @@ For publicly publishing a port, the conlink container needs to be on
 a docker network and the `conlink_port` should match the target port
 of a docker published port (for the conlink container).
 
+For the `ethtool` property, refer to the `ethtool` man page. The
+syntax for each ethtool setting is basically the ethtool command line
+arguments without the "devname. So the equivalent of the ethtool
+command `ethtool --offload eth0 rx off` would be link configuration
+`{dev: eth0, ethtool: ["--offload rx off"], ...}`.
+
 ### Bridges
 
 The bridge settings currently only support the "mode" setting. If

examples/test7-compose.yaml

@@ -29,6 +29,7 @@ services:
           mac: 00:0a:0b:0c:0d:01
           mtu: 4111
           netem: "rate 10mbit delay 40ms"
+          ethtool: "--offload rx off"
         - bridge: s2
           ip: 100.0.1.1/16
           dev: eth1

link-add.sh

@@ -42,7 +42,10 @@ usage () {
   echo >&2 "  --remote REMOTE          - Remote address for geneve/vxlan types"
   echo >&2 "  --vni VNI                - Virtual Network Identifier for geneve/vxlan types"
   echo >&2 ""
-  echo >&2 "  --netem NETEM            - tc qdisc netem OPTIONS (man 8 netem) (can repeat)"
+  echo >&2 "  --netem NETEM            - tc qdisc netem OPTIONS (can repeat)"
+  echo >&2 "                             (man 8 netem)"
+  echo >&2 "  --ethtool 'ARG OPTS'     - ethtool ARG INTF0 OPTS (can repeat)"
+  echo >&2 "                             (man 8 ethtool)"
   echo >&2 "  --nat TARGET             - Stateless NAT traffic to/from TARGET"
   echo >&2 "                             (in primary/PID0 netns)"
   echo >&2 ""
@@ -58,16 +61,16 @@ setup_if() {
   local IF=$1 NS=$2 MAC=$3 IP=$4 MTU=$5 ROUTES=$6 routes=
   echo >&2 "ROUTES: ${ROUTES}"
   while read rt; do
-      [ "${rt}" ] && routes="${routes}\nroute add ${rt} dev ${IF}"
-  done < <(echo -e "${ROUTES}")
+      routes="${routes}route add ${rt} dev ${IF}\n"
+  done < <(echo -en "${ROUTES}")
 
   info "Setting ${IP:+IP ${IP}, }${MAC:+MAC ${MAC}, }${MTU:+MTU ${MTU}, }${ROUTES:+ROUTES '${ROUTES//$'\n'/,}', }up state"
   ip -netns ${NS} --force -b - <<EOF
       ${IP:+addr add ${IP} dev ${IF}}
       ${MAC:+link set dev ${IF} address ${MAC}}
       ${MTU:+link set dev ${IF} mtu ${MTU}}
       link set dev ${IF} up
-      $(echo -e "${routes}")
+      $(echo -en "${routes}")
 EOF
 }
 
@@ -82,7 +85,7 @@ IPTABLES() {
 VERBOSE=${VERBOSE:-}
 PID1=${PID1:-<SELF>} IF1=${IF1:-eth0}
 IP0= IP1= MAC0= MAC1= ROUTES0= ROUTES1= MTU=
-MODE= VLANID= REMOTE= VNI= NETEM= NAT=
+MODE= VLANID= REMOTE= VNI= NETEM= NAT= ETHTOOL=
 positional=
 while [ "${*}" ]; do
   param=$1; OPTARG=$2
@@ -94,10 +97,10 @@ while [ "${*}" ]; do
   --ip1)            IP1="${OPTARG}"; shift ;;
   --mac|--mac0)     MAC0="${OPTARG}"; shift ;;
   --mac1)           MAC1="${OPTARG}"; shift ;;
-  --route|--route0) ROUTES0="${ROUTES0}\n${OPTARG}"; shift ;;
-  --route1)         ROUTES1="${ROUTES1}\n${OPTARG}"; shift ;;
+  --route|--route0) ROUTES0="${ROUTES0}${OPTARG}\n"; shift ;;
+  --route1)         ROUTES1="${ROUTES1}${OPTARG}\n"; shift ;;
   --mtu)            MTU="${OPTARG}"; shift ;;
-
+  --ethtool)        ETHTOOL="${ETHTOOL}${OPTARG}\n"; shift ;;
   --mode)           MODE="${OPTARG}"; shift ;;
   --vlanid)         VLANID="${OPTARG}"; shift ;;
 
@@ -111,8 +114,6 @@ while [ "${*}" ]; do
   esac
   shift
 done
-ROUTES0="${ROUTES0#\\n}"
-ROUTES1="${ROUTES1#\\n}"
 set -- ${positional}
 TYPE=$1 PID0=$2 IF0=$3
 
@@ -195,6 +196,11 @@ if [ "${NETEM}" ]; then
   tc -netns ${NS0} qdisc add dev ${IF0} root netem ${NETEM}
 fi
 
+while read arg opts; do
+  info "Applying ethtool ${arg} ${IF0} ${opts} (in ${NS0})"
+  ip netns exec ${NS0} ethtool ${arg} ${IF0} ${opts}
+done < <(echo -en "${ETHTOOL}")
+
 if [ "${NAT}" ]; then
   info "Adding NAT rule to ${NAT}"
   IPTABLES ${NS0} PREROUTING  -t nat -i ${IF0} -j DNAT --to-destination ${NAT}

schema.yaml

@@ -53,6 +53,9 @@ properties:
         netem:
           oneOf: [{type: string},
                   {type: array, items: {type: string}}]
+        ethtool:
+          oneOf: [{type: string},
+                  {type: array, items: {type: string}}]
 
   bridges:
     type: array

src/conlink/core.cljs

@@ -32,6 +32,8 @@ General Options:
                                     [default: auto] [env: CONLINK_BRIDGE_MODE]
   --default-mtu MTU                 Default link MTU (for non *vlan types)
                                     [default: 65535]
+  --keep-veth-offload               Do not add '--offload tx off' as the first
+                                    ethtool setting for container veth interfaces
   --network-file NETWORK-FILE...    Network config file
   --compose-file COMPOSE-FILE...    Docker compose file with network config
   --compose-project NAME            Docker compose project name for resolving
@@ -57,7 +59,8 @@ General Options:
                         " --system-id=random --no-mlockall --delete-bridges"))
 
 (def VLAN-TYPES #{:vlan :macvlan :macvtap :ipvlan :ipvtap})
-(def LINK-ADD-OPTS [:ip :mac :route :mtu :nat :netem :mode :vlanid :remote :vni])
+(def LINK-ADD-OPTS [:ip :mac :route :mtu :nat :netem :ethtool
+                    :mode :vlanid :remote :vni])
 (def INTF-MAX-LEN 15)
 (def DOCKER-INTF "DOCKER-ETH0")
 
@@ -112,9 +115,9 @@ General Options:
     - mac: random MAC starting with first octet of 'c2'
     - mtu: --default-mtu (for non *vlan type)
     - base: :conlink for veth type, :host for *vlan types, :local otherwise"
-  [{:as link :keys [type base bridge ip route forward netem]} bridges opts]
+  [{:as link :keys [type bridge ip route forward netem ethtool]} bridges opts]
   (let [{:keys [docker-eth0? docker-eth0-address]} @ctx
-        {:keys [default-mtu]} opts
+        {:keys [default-mtu keep-veth-offload]} opts
         type (keyword (or type "veth"))
         dev (get link :dev "eth0")
         mac (get link :mac (random-mac))
@@ -126,12 +129,17 @@ General Options:
         route (if (string? route) [route] route)
         forward (if (string? forward) [forward] forward)
         netem (if (string? netem) [netem] netem)
+        ethtool-pre (if (and (= :veth type) (not keep-veth-offload))
+                      ["--offload tx off"]
+                      [])
+        ethtool (into ethtool-pre (if (string? ethtool) [ethtool] ethtool))
         link (merge
                link
                {:type type
                 :dev  dev
                 :base base
-                :mac mac}
+                :mac mac
+                :ethtool ethtool}
                (when bridge
                  {:bridge bridge})
                (when (not (VLAN-TYPES type))