Network churn Load test - Add network policy enforcement latency measurement

jobs/competitive-test.yml

@@ -33,6 +33,9 @@ parameters:
 - name: run_id
   type: string
   default: ''
+- name: run_id_2
+  type: string
+  default: ''
 - name: timeout_in_minutes
   type: number
   default: 60 # default when not specified is 60 minutes
@@ -48,6 +51,9 @@ parameters:
 - name: ssh_key_enabled
   type: boolean
   default: true
+- name: use_secondary_cluster
+  type: boolean
+  default: false
 
 jobs:
 - job: ${{ parameters.cloud }}
@@ -62,10 +68,12 @@ jobs:
       cloud: ${{ parameters.cloud }}
       region: ${{ parameters.regions[0] }}
       run_id: ${{ parameters.run_id }}
+      run_id_2: ${{ parameters.run_id_2 }}
       test_modules_dir: ${{ parameters.test_modules_dir }}
       retry_attempt_count: ${{ parameters.retry_attempt_count }}
       credential_type: ${{ parameters.credential_type }}
       ssh_key_enabled: ${{ parameters.ssh_key_enabled }}
+      use_secondary_cluster: ${{ parameters.use_secondary_cluster }}
   - template: /steps/provision-resources.yml
     parameters:
       cloud: ${{ parameters.cloud }}

modules/python/clusterloader2/slo/config/deployment_template.yaml

@@ -3,6 +3,13 @@
 
 {{$Image := DefaultParam .Image "mcr.microsoft.com/oss/kubernetes/pause:3.6"}}
 
+{{$EnableNetworkPolicyEnforcementLatencyTest := DefaultParam .EnableNetworkPolicyEnforcementLatencyTest false}}
+{{$TargetLabelValue := DefaultParam .TargetLabelValue "enforcement-latency"}}
+# Run a server pod for network policy enforcement latency test only on every Nth pod.
+# Default every third pod.
+{{$NetPolServerOnEveryNthPod := 3}}
+{{$RunNetPolicyTest := and $EnableNetworkPolicyEnforcementLatencyTest (eq (Mod .Index $NetPolServerOnEveryNthPod) 0)}}
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -16,7 +23,7 @@ spec:
   replicas: {{.Replicas}}
   selector:
     matchLabels:
-      name: {{.Name}}
+      name: {{if $RunNetPolicyTest}}policy-load-{{end}}{{.Name}}
   strategy:
     type: RollingUpdate
     rollingUpdate:
@@ -25,29 +32,43 @@ spec:
   template:
     metadata:
       labels:
-        name: {{.Name}}
+        name: {{if $RunNetPolicyTest}}policy-load-{{end}}{{.Name}}
         group: {{.Group}}
 {{if .SvcName}}
         svc: {{.SvcName}}-{{.Index}}
 {{end}}
         restart: {{.deploymentLabel}}
+{{if $RunNetPolicyTest}}
+        net-pol-test: {{$TargetLabelValue}}
+{{end}}
     spec:
       nodeSelector:
         slo: "true"
+{{if $RunNetPolicyTest}}
+      hostNetwork: false
+      containers:
+      - image: nginx
+        name: nginx-server
+        ports:
+        - containerPort: 80
+        resources:
+          requests:
+            cpu: {{$CpuRequest}}
+            memory: {{$MemoryRequest}}
+{{else}}
       containers:
       - env:
         - name: ENV_VAR
           value: a
         image: {{$Image}}
         imagePullPolicy: IfNotPresent
         name: {{.Name}}
-        ports:
+        ports: []
         resources:
           requests:
             cpu: {{$CpuRequest}}
             memory: {{$MemoryRequest}}
-      # Add not-ready/unreachable tolerations for 15 minutes so that node
-      # failure doesn't trigger pod deletion.
+{{end}}
       tolerations:
       - key: "node.kubernetes.io/not-ready"
         operator: "Exists"
@@ -60,4 +81,4 @@ spec:
       - key: "slo"
         operator: "Equal"
         value: "true"
-        effect: "NoSchedule"
+        effect: "NoSchedule"

modules/python/clusterloader2/slo/config/load-config.yaml

@@ -2,6 +2,7 @@ name: load-config
 
 # Config options for test type
 {{$SERVICE_TEST := DefaultParam .CL2_SERVICE_TEST true}}
+{{$NETWORK_TEST := DefaultParam .CL2_NETWORK_TEST false}}
 
 # Config options for test parameters
 {{$nodesPerNamespace := DefaultParam .CL2_NODES_PER_NAMESPACE 100}}
@@ -12,12 +13,12 @@ name: load-config
 {{$groupName := DefaultParam .CL2_GROUP_NAME "service-discovery"}}
 
 # TODO(jshr-w): This should eventually use >1 namespace.
-{{$namespaces := 1}}
+{{$namespaces := DefaultParam .CL2_NO_OF_NAMESPACES 1}}
 {{$nodes := DefaultParam .CL2_NODES 1000}}
 
 {{$deploymentQPS := DivideFloat $loadTestThroughput $deploymentSize}}
 {{$operationTimeout := DefaultParam .CL2_OPERATION_TIMEOUT "15m"}}
-{{$totalPods := MultiplyInt $namespaces $nodes $podsPerNode}}
+{{$totalPods := MultiplyInt $namespaces $nodesPerNamespace $podsPerNode}}
 {{$podsPerNamespace := DivideInt $totalPods $namespaces}}
 {{$deploymentsPerNamespace := DivideInt $podsPerNamespace $deploymentSize}}
 
@@ -29,17 +30,17 @@ name: load-config
 
 # Service test
 {{$BIG_GROUP_SIZE := DefaultParam .BIG_GROUP_SIZE 4000}}
-{{$SMALL_GROUP_SIZE := DefaultParam .SMALL_GROUP_SIZE 20}}
+{{$SMALL_GROUP_SIZE := DefaultParam .CL2_DEPLOYMENT_SIZE 20}}
 {{$bigDeploymentsPerNamespace := DefaultParam .bigDeploymentsPerNamespace 1}}
-{{$smallDeploymentPods := SubtractInt $podsPerNamespace (MultiplyInt $bigDeploymentsPerNamespace $BIG_GROUP_SIZE)}}
+{{$smallDeploymentPods :=  DivideInt $totalPods $namespaces}}
 {{$smallDeploymentsPerNamespace := DivideInt $smallDeploymentPods $SMALL_GROUP_SIZE}}
 
 namespace:
   number: {{$namespaces}}
   prefix: slo
   deleteStaleNamespaces: true
   deleteAutomanagedNamespaces: true
-  enableExistingNamespaces: false
+  enableExistingNamespaces: true
 
 tuningSets:
   - name: Sequence
@@ -53,7 +54,7 @@ tuningSets:
       qps: {{$deploymentQPS}}
 
 steps:
-  - name: Log - namespaces={{$namespaces}}, nodesPerNamespace={{$nodesPerNamespace}}, podsPerNode={{$podsPerNode}}, totalPods={{$totalPods}}, podsPerNamespace={{$podsPerNamespace}}, deploymentsPerNamespace={{$deploymentsPerNamespace}}, deploymentSize={{$deploymentSize}}, deploymentQPS={{$deploymentQPS}}
+  - name: Log - namespaces={{$namespaces}}, nodes={{$nodes}}, nodesPerNamespace={{$nodesPerNamespace}}, podsPerNode={{$podsPerNode}}, totalPods={{$totalPods}}, podsPerNamespace={{$podsPerNamespace}}, deploymentsPerNamespace={{$deploymentsPerNamespace}}, deploymentSize={{$deploymentSize}}, deploymentQPS={{$deploymentQPS}}
     measurements:
     - Identifier: Dummy
       Method: Sleep
@@ -74,6 +75,13 @@ steps:
         action: start
 {{end}}
 
+{{if $NETWORK_TEST}}
+  - module:
+        path: /modules/network-policy/net-policy-metrics.yaml
+        params:
+          action: start
+{{end}}
+
 {{range $i := Loop $repeats}}
 {{if $SERVICE_TEST}}
   - module:
@@ -85,6 +93,15 @@ steps:
         bigServicesPerNamespace: {{$bigDeploymentsPerNamespace}}
 {{end}}
 
+{{if $NETWORK_TEST}}
+  - module:
+      path: modules/network-policy/net-policy-enforcement-latency.yaml
+      params:
+        setup: true
+        run: true
+        testType: "pod-creation"
+{{end}}
+
   - module:
       path: /modules/reconcile-objects.yaml
       params:
@@ -101,6 +118,27 @@ steps:
         Group: {{$groupName}}
         deploymentLabel: start
 
+{{if $NETWORK_TEST}}
+  - module:
+      path: modules/network-policy/net-policy-metrics.yaml
+      params:
+        action: gather
+        usePolicyCreationMetrics: true
+        usePodCreationMetrics: true
+
+  - module:
+      path: modules/network-policy/net-policy-enforcement-latency.yaml
+      params:
+        complete: true
+        testType: "pod-creation"
+
+  - module:
+      path: modules/network-policy/net-policy-enforcement-latency.yaml
+      params:
+        run: true
+        testType: "policy-creation"
+{{end}}
+
   - module:
       path: /modules/reconcile-objects.yaml
       params:
@@ -152,3 +190,18 @@ steps:
       params:
         action: gather
         group: {{$groupName}}
+
+{{if $NETWORK_TEST}}
+  - module:
+      path: modules/network-policy/net-policy-metrics.yaml
+      params:
+        action: gather
+        usePolicyCreationMetrics: true
+        usePodCreationMetrics: true
+
+  - module:
+      path: modules/network-policy/net-policy-enforcement-latency.yaml
+      params:
+        complete: true
+        testType: "policy-creation"
+{{end}}

modules/python/clusterloader2/slo/config/modules/network-policy/net-policy-enforcement-latency.yaml

@@ -0,0 +1,57 @@
+{{$NETWORK_POLICY_ENFORCEMENT_LATENCY_BASELINE := DefaultParam .CL2_NETWORK_POLICY_ENFORCEMENT_LATENCY_BASELINE false}}
+{{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_KEY := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_KEY "net-pol-test"}}
+{{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE "enforcement-latency"}}
+{{$NET_POLICY_ENFORCEMENT_LATENCY_NODE_LABEL_KEY := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LATENCY_NODE_LABEL_KEY "test"}}
+{{$NET_POLICY_ENFORCEMENT_LATENCY_NODE_LABEL_VALUE := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LATENCY_NODE_LABEL_VALUE "net-policy-client"}}
+{{$NET_POLICY_ENFORCEMENT_LATENCY_MAX_TARGET_PODS_PER_NS := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LATENCY_MAX_TARGET_PODS_PER_NS 100}}
+{{$NET_POLICY_ENFORCEMENT_LOAD_COUNT := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LOAD_COUNT 1000}}
+{{$NET_POLICY_ENFORCEMENT_LOAD_QPS := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LOAD_QPS 10}}
+{{$NET_POLICY_ENFORCEMENT_LOAD_TARGET_NAME := DefaultParam .CL2_POLICY_ENFORCEMENT_LOAD_TARGET_NAME "small-deployment"}}
+
+{{$setup := DefaultParam .setup false}}
+{{$run := DefaultParam .run false}}
+{{$complete := DefaultParam .complete false}}
+{{$testType := DefaultParam .testType "policy-creation"}}
+# Target port needs to match the server container port of target pods that have
+# "targetLabelKey: targetLabelValue" label selector.
+{{$targetPort := 80}}
+
+steps:
+  {{if $setup}}
+- name: Setup network policy enforcement latency measurement
+  measurements:
+  - Identifier: NetworkPolicyEnforcement
+    Method: NetworkPolicyEnforcement
+    Params:
+      action: setup
+      targetLabelKey: {{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_KEY}}
+      targetLabelValue: {{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE}}
+      baseline: {{$NETWORK_POLICY_ENFORCEMENT_LATENCY_BASELINE}}
+      testClientNodeSelectorKey: {{$NET_POLICY_ENFORCEMENT_LATENCY_NODE_LABEL_KEY}}
+      testClientNodeSelectorValue: {{$NET_POLICY_ENFORCEMENT_LATENCY_NODE_LABEL_VALUE}}
+  {{end}}
+
+  {{if $run}}
+- name: "Run pod creation network policy enforcement latency measurement (testType={{$testType}})"
+  measurements:
+  - Identifier: NetworkPolicyEnforcement
+    Method: NetworkPolicyEnforcement
+    Params:
+      action: run
+      testType: {{$testType}}
+      targetPort: {{$targetPort}}
+      maxTargets: {{$NET_POLICY_ENFORCEMENT_LATENCY_MAX_TARGET_PODS_PER_NS}}
+      policyLoadCount: {{$NET_POLICY_ENFORCEMENT_LOAD_COUNT}}
+      policyLoadQPS: {{$NET_POLICY_ENFORCEMENT_LOAD_QPS}}
+      policyLoadTargetBaseName: {{$NET_POLICY_ENFORCEMENT_LOAD_TARGET_NAME}}
+  {{end}}
+
+  {{if $complete}}
+- name: "Complete pod creation network policy enforcement latency measurement (testType={{$testType}})"
+  measurements:
+  - Identifier: NetworkPolicyEnforcement
+    Method: NetworkPolicyEnforcement
+    Params:
+      action: complete
+      testType: {{$testType}}
+  {{end}}