Putting benchmark comparison in utils

utils/common.sh

@@ -12,7 +12,7 @@ log() {
 function openshift_login () {
   if [[ -z $KUBECONFIG ]] && [[ ! -s $HOME/.kube/config ]]; then
     log "KUBECONFIG var is not defined and cannot find kube config in the home directory, trying to use oc login"
-    if [[ -n ${KUBEUSER}} ]] && [[ -n ${KUBEPASSWORD} ]] && [[ -n ${KUBEURL} ]]; then
+    if [[ -n ${KUBEUSER} ]] && [[ -n ${KUBEPASSWORD} ]] && [[ -n ${KUBEURL} ]]; then
       oc login -u ${KUBEUSER} -p ${KUBEPASSWORD} ${KUBEURL}
     else
      log "No openshift authentication method found, exiting"
@@ -76,9 +76,6 @@ function check_pod_ready_state () {
   return $?
 }
 
-
-
-
 gen_spreadsheet_helper() {
   pip install oauth2client>=4.1.3 gspread
   sheetname=${1}-$(date "+%Y-%m-%dT%H:%M:%S")

utils/compare.sh

@@ -2,6 +2,72 @@
 #
 # Handles benchmark-comparison execution
 
+source common.sh
+
+get_network_type() {
+if [[ $NETWORK_TYPE == "OVNKubernetes" ]]; then
+  network_ns=openshift-ovn-kubernetes
+else
+  network_ns=openshift-sdn
+fi
+echo $network_ns
+}
+
+check_metric_to_modify() {
+  export div_by=1
+  declare file_content=$( cat "${1}" )
+  if [[ $file_content =~ "memory" ]]; then
+   export div_by=1048576
+  fi
+  if [[ $file_content =~ "latency" ]]; then
+    export div_by=1000
+  fi
+  if [[ $file_content =~ "byte" ]]; then
+    export div_by=1000000
+  fi
+}
+
+run_benchmark_comparison() {
+   log "benchmark"
+   if [[ -n ${ES_SERVER} ]] && [[ -n ${COMPARISON_CONFIG} ]]; then
+
+     log "Installing touchstone"
+     install_touchstone
+     network_ns=openshift-ovn-kubernetes
+     get_network_type
+     export TOUCHSTONE_NAMESPACE=${TOUCHSTONE_NAMESPACE:-"$network_ns"}
+     res_output_dir="/tmp/${WORKLOAD}-${UUID}"
+     mkdir -p ${res_output_dir}
+     export COMPARISON_OUTPUT=${PWD}/${WORKLOAD}-${UUID}.csv
+     final_csv=${res_output_dir}/${UUID}.csv
+     echo "final csv $final_csv"
+     for config in ${COMPARISON_CONFIG}
+     do
+       config_loc=$(dirname $(realpath ${BASH_SOURCE[0]}))/touchstone-configs/${config}
+       echo "config ${config_loc}"
+       check_metric_to_modify $config_loc
+       COMPARISON_FILE="${res_output_dir}/${config}"
+       envsubst < $config_loc > $COMPARISON_FILE
+       echo "comparison output"
+       if [[ -n ${ES_SERVER_BASELINE} ]] && [[ -n ${BASELINE_UUID} ]]; then
+         log "Comparing with baseline"
+         compare "${ES_SERVER_BASELINE} ${ES_SERVER}" "${BASELINE_UUID} ${UUID}" "${COMPARISON_FILE}" "${GEN_CSV}"
+       else
+         log "Querying results"
+         compare ${ES_SERVER} ${UUID} "${COMPARISON_FILE}" "${GEN_CSV}"
+       fi
+        log "python csv modifier"
+        python $(dirname $(realpath ${BASH_SOURCE[0]}))/csv_modifier.py -c ${COMPARISON_OUTPUT} -o ${final_csv}
+     done
+     if [[ -n ${GSHEET_KEY_LOCATION} ]] && [[ -n ${COMPARISON_OUTPUT} ]]; then
+       echo "sheet $GSHEET_KEY_LOCATION "
+       gen_spreadsheet ${WORKLOAD} ${final_csv} ${EMAIL_ID_FOR_RESULTS_SHEET} ${GSHEET_KEY_LOCATION}
+     fi
+     log "Removing touchstone"
+     remove_touchstone
+  fi
+ }
+
 
 install_touchstone() {
   touchstone_tmp=$(mktemp -d)
@@ -43,4 +109,4 @@ compare() {
   fi
   log "Running: ${cmd}"
   ${cmd}
-}
+}

utils/csv_modifier.py

@@ -1,6 +1,5 @@
 import pandas as pd
 from typing import *
-import sys
 import argparse
 import os
 
@@ -38,7 +37,10 @@ def read_csv(filepath: str) -> pd.DataFrame:
 
 def divide_col(x: float) -> float:
     div_by = os.environ.get("div_by", 1)
-    return round(x / float((div_by)), 3)
+    try: 
+        return round(x / float((div_by)), 3)
+    except:
+        return x
 
 
 def write_df(df: pd.DataFrame, target_filepath: str, mode: str = "a"):
@@ -57,7 +59,8 @@ def main():
             select_columns = -1
             if "ES_SERVER_BASELINE" in os.environ and "BASELINE_UUID" in os.environ:
                 select_columns = -2
-            df.sort_values(df.columns[-1], ascending=False, inplace=True)
+            if ("SORT_BY_VALUE" in os.environ) and (os.getenv("SORT_BY_VALUE") == "true"):
+                df.sort_values(df.columns[-1], ascending=False, inplace=True)
             df[df.columns[select_columns:num_columns]] = df[
                 df.columns[select_columns:num_columns]
             ].apply(divide_col)
@@ -67,4 +70,4 @@ def main():
         )
 
 
-main()
+main()

utils/touchstone-configs/uperf-touchstone-norm-ltcy.json

@@ -0,0 +1,25 @@
+{
+  "elasticsearch": {
+    "ripsaw-uperf-results": [
+      {
+        "filter": {
+          "test_type.keyword": "rr"
+        },
+        "exclude": {
+          "norm_ops": 0
+        },
+        "buckets": [
+          "protocol.keyword",
+          "message_size",
+          "num_threads",
+          "num_pairs.keyword"
+        ],
+        "aggregations": {
+          "norm_ltcy": [
+            "avg"
+          ]
+        }
+      }
+    ]
+  }
+}

utils/touchstone-configs/uperf-touchstone-norm-ops.json

@@ -0,0 +1,25 @@
+{
+  "elasticsearch": {
+    "ripsaw-uperf-results": [
+      {
+        "filter": {
+          "test_type.keyword": "rr"
+        },
+        "exclude": {
+          "norm_ops": 0
+        },
+        "buckets": [
+          "protocol.keyword",
+          "message_size",
+          "num_threads",
+          "num_pairs.keyword"
+        ],
+        "aggregations": {
+          "norm_ops": [
+            "avg"
+          ]
+        }
+      }
+    ]
+  }
+}

utils/touchstone-configs/uperf-touchstone-rr.json

@@ -0,0 +1,31 @@
+{
+  "elasticsearch": {
+    "ripsaw-uperf-results": [
+      {
+        "filter": {
+          "test_type.keyword": "rr"
+        },
+        "exclude": {
+          "norm_ops": 0
+        },
+        "buckets": [
+          "protocol.keyword",
+          "message_size",
+          "num_threads",
+          "num_pairs.keyword"
+        ],
+        "aggregations": {
+          "norm_ltcy": [
+            {
+              "percentiles": {
+                "percents": [
+                  99
+                ]
+              }
+            }
+          ]
+        }
+      }
+    ]
+  }
+}

utils/touchstone-configs/uperf-touchstone-stream.json

@@ -0,0 +1,25 @@
+{
+  "elasticsearch": {
+    "ripsaw-uperf-results": [
+      {
+        "filter": {
+          "test_type.keyword": "stream"
+        },
+        "exclude": {
+          "norm_ops": 0
+        },
+        "buckets": [
+          "protocol.keyword",
+          "message_size",
+          "num_threads",
+          "num_pairs.keyword"
+        ],
+        "aggregations": {
+          "norm_byte": [
+            "avg"
+          ]
+        }
+      }
+    ]
+  }
+}

workloads/kube-burner/README.md

@@ -55,7 +55,7 @@ Workloads can be tweaked with the following environment variables:
 | **THANOS_RECEIVER_URL** | Thanos receiver url endpoint for grafana remote-write agent |  | 
 | **POD_READY_THRESHOLD** | Pod ready latency threshold (only applies node-density and pod-density workloads). [More info](https://kube-burner.readthedocs.io/en/latest/measurements/#pod-latency-thresholds) | 5000ms |
 | **PLATFORM_ALERTS** | Platform alerting enables, kube-burner alerting based cluster's platform, either ALERT_PROFILE or this variable can be set | false |
-| **COMPARISON_CONFIG**        | Touchstone configs. Multiple config files can be passed here. Ex. COMPARISON_CONFIG="podCPU-avg.json clusterVersion.json". [Sample files](https://github.com/cloud-bulldozer/e2e-benchmarking/tree/master/workloads/kube-burner/touchstone-configs) |  |
+| **COMPARISON_CONFIG**        | Touchstone configs. Multiple config files can be passed here. Ex. COMPARISON_CONFIG="podCPU-avg.json clusterVersion.json". [Sample files](../../utils/touchstone-configs) |  |
 | **TOUCHSTONE_NAMESPACE**        | Namespace where we query for metrics specified in touchstone config files | openshift-sdn or openshift-ovn-kubernetes |
 | **GSHEET_KEY_LOCATION**        | Location of service account key to generate google sheets |  |
 | **EMAIL_ID_FOR_RESULTS_SHEET**        | Email id where the google sheets needs to be sent |  |

workloads/kube-burner/common.sh

@@ -2,7 +2,7 @@
 set -m
 source ../../utils/common.sh
 source env.sh
-source ../../utils/compare.sh
+
 
 openshift_login
 
@@ -132,58 +132,6 @@ delete_oldpprof_folder() {
  rm -rf pprof-data
 }
 
-get_network_type() {
-if [[ $NETWORK_TYPE == "OVNKubernetes" ]]; then
-  network_ns=openshift-ovn-kubernetes
-else
-  network_ns=openshift-sdn
-fi
-}
-
-check_metric_to_modify() {
-  export div_by=1
-  echo $config | grep -i memory
-  if [[ $? == 0 ]]; then 
-   export div_by=1048576
-  fi
-  echo $config | grep -i latency
-  if [[ $? == 0 ]]; then
-    export div_by=1000
-  fi
-}
-
-run_benchmark_comparison() {
-   if [[ -n ${ES_SERVER} ]] && [[ -n ${COMPARISON_CONFIG} ]]; then
-     log "Installing touchstone"
-     install_touchstone
-     get_network_type
-     export TOUCHSTONE_NAMESPACE=${TOUCHSTONE_NAMESPACE:-"$network_ns"}
-     res_output_dir="/tmp/${WORKLOAD}-${UUID}"
-     mkdir -p ${res_output_dir}
-     final_csv=${res_output_dir}/${UUID}.csv
-     for config in ${COMPARISON_CONFIG}; do
-       check_metric_to_modify
-       envsubst < touchstone-configs/${config} > /tmp/${config}
-       COMPARISON_OUTPUT="${res_output_dir}/${config}"
-       if [[ -n ${ES_SERVER_BASELINE} ]] && [[ -n ${BASELINE_UUID} ]]; then
-         log "Comparing with baseline"
-         compare "${ES_SERVER_BASELINE} ${ES_SERVER}" "${BASELINE_UUID} ${UUID}" "/tmp/${config}" "${GEN_CSV}"
-       else
-         log "Querying results"
-         compare ${ES_SERVER} ${UUID} "/tmp/${config}" "${GEN_CSV}"
-       fi
-       if [[ ${GEN_CSV} == "true" ]]; then
-         python ../../utils/csv_modifier.py -c ${COMPARISON_OUTPUT} -o ${final_csv}
-       fi
-     done
-     if [[ -n ${GSHEET_KEY_LOCATION} ]] && [[ ${GEN_CSV} == "true" ]]; then
-       gen_spreadsheet ${WORKLOAD} ${final_csv} ${EMAIL_ID_FOR_RESULTS_SHEET} ${GSHEET_KEY_LOCATION}
-     fi
-     log "Removing touchstone"
-     remove_touchstone
-  fi
- }
-
 label_node_with_label() {
   colon_param=$(echo $1 | tr "=" ":" | sed 's/:/: /g')
   export POD_NODE_SELECTOR="{$colon_param}"

workloads/kube-burner/env.sh

@@ -82,4 +82,5 @@ export PLATFORM_ALERTS=${PLATFORM_ALERTS:-false}
 export COMPARISON_CONFIG=${COMPARISON_CONFIG:-""}
 export GSHEET_KEY_LOCATION=${GSHEET_KEY_LOCATION}
 export EMAIL_ID_FOR_RESULTS_SHEET=${EMAIL_ID_FOR_RESULTS_SHEET}
-GEN_CSV=${GEN_CSV:-false}
+export GEN_CSV=${GEN_CSV:-false}
+export SORT_BY_VALUE=${SORT_BY_VALUE:-true}

workloads/kube-burner/run.sh

@@ -2,6 +2,7 @@
 
 . common.sh
 . build_helper.sh
+. ../../utils/compare.sh
 
 label=""
 case ${WORKLOAD} in

workloads/network-perf/README.md

@@ -48,12 +48,13 @@ The environment variables below are used to configure benchmark comparison and/o
 | **COMPARISON_ALIASES**  | Benchmark-comparison aliases (UUIDs will be replaced by these aliases | "" |
 | **ES_SERVER_BASELINE**  | Elasticsearch endpoint used used by the baseline benchmark | https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 |
 | **BASELINE_UUID**       | Baseline UUID used for comparison | "" |
-| **COMPARISON_CONFIG**   | Benchmark-comparison config file | `${PWD}/uperf-touchstone.json` |
+| **COMPARISON_CONFIG**   | Benchmark-comparison config file,, file needs to be in **e2e-benchmarking/utils/touchstone-config **folder | [uperf-touchstone.json](../../utils/touchstone-configs/uperf-touchstone.json)|
 | **COMPARISON_RC**       | Benchmark-comparison return code if tolerancy check fails | 0 |
 | **TOLERANCY_RULES_CFG** | Tolerancy rules configuration file | uperf-tolerancy-rules.yaml |
 | **GSHEET_KEY_LOCATION** | Location of the Google Service Account Key, used to import a resulting csv | "" |
 | **EMAIL_ID_FOR_RESULTS_SHEET**   | Email to push CSV results | "" |
 | **GEN_CSV**             | Generate a benchmark-comparison csv, required to generate the spreadsheet | "false" |
+| **SORT_BY_VALUE**             | Sort the generated CSV by value | false |
 
 ## Snappy integration configurations
 

workloads/network-perf/common.sh

@@ -3,7 +3,6 @@
 source env.sh
 source ../../utils/common.sh
 source ../../utils/benchmark-operator.sh
-source ../../utils/compare.sh
 
 openshift_login
 
@@ -23,24 +22,5 @@ deploy_operator() {
   deploy_benchmark_operator
 }
 
-run_benchmark_comparison() {
-  if [[ -n ${ES_SERVER} ]]; then
-    log "Installing touchstone"
-    install_touchstone
-    if [[ -n ${ES_SERVER_BASELINE} ]] && [[ -n ${BASELINE_UUID} ]]; then
-      log "Comparing with baseline"
-      compare "${ES_SERVER_BASELINE} ${ES_SERVER}" "${BASELINE_UUID} ${UUID}" ${COMPARISON_CONFIG} ${GEN_CSV}
-    else
-      log "Querying results"
-      compare ${ES_SERVER} ${UUID} ${COMPARISON_CONFIG} ${GEN_CSV}
-    fi
-    if [[ -n ${GSHEET_KEY_LOCATION} ]] && [[ ${GEN_CSV} == "true" ]]; then
-      gen_spreadsheet network-performance ${COMPARISON_OUTPUT} ${EMAIL_ID_FOR_RESULTS_SHEET} ${GSHEET_KEY_LOCATION}
-    fi
-    log "Removing touchstone"
-    remove_touchstone
-  fi
-}
-
 export_defaults
 deploy_operator