Skip to content

Commit 8223a4f

Browse files
paigerube14paigerube14
committed
putting benchmark comparison in utils
1 parent 6e3206a commit 8223a4f

29 files changed

+200
-110
lines changed

‎utils/common.sh

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ log() {
1212
function openshift_login () {
1313
if [[ -z $KUBECONFIG ]] && [[ ! -s $HOME/.kube/config ]]; then
1414
log "KUBECONFIG var is not defined and cannot find kube config in the home directory, trying to use oc login"
15-
if [[ -n ${KUBEUSER}} ]] && [[ -n ${KUBEPASSWORD} ]] && [[ -n ${KUBEURL} ]]; then
15+
if [[ -n ${KUBEUSER} ]] && [[ -n ${KUBEPASSWORD} ]] && [[ -n ${KUBEURL} ]]; then
1616
oc login -u ${KUBEUSER} -p ${KUBEPASSWORD} ${KUBEURL}
1717
else
1818
log "No openshift authentication method found, exiting"
@@ -76,9 +76,6 @@ function check_pod_ready_state () {
7676
return $?
7777
}
7878

79-
80-
81-
8279
gen_spreadsheet_helper() {
8380
pip install oauth2client>=4.1.3 gspread
8481
sheetname=${1}-$(date "+%Y-%m-%dT%H:%M:%S")

‎utils/compare.sh

100644100755
Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,72 @@
22
#
33
# Handles benchmark-comparison execution
44

5+
source common.sh
6+
7+
get_network_type() {
8+
if [[ $NETWORK_TYPE == "OVNKubernetes" ]]; then
9+
network_ns=openshift-ovn-kubernetes
10+
else
11+
network_ns=openshift-sdn
12+
fi
13+
echo $network_ns
14+
}
15+
16+
check_metric_to_modify() {
17+
export div_by=1
18+
declare file_content=$( cat "${1}" )
19+
if [[ $file_content =~ "memory" ]]; then
20+
export div_by=1048576
21+
fi
22+
if [[ $file_content =~ "latency" ]]; then
23+
export div_by=1000
24+
fi
25+
if [[ $file_content =~ "byte" ]]; then
26+
export div_by=1000000
27+
fi
28+
}
29+
30+
run_benchmark_comparison() {
31+
log "benchmark"
32+
if [[ -n ${ES_SERVER} ]] && [[ -n ${COMPARISON_CONFIG} ]]; then
33+
34+
log "Installing touchstone"
35+
install_touchstone
36+
network_ns=openshift-ovn-kubernetes
37+
get_network_type
38+
export TOUCHSTONE_NAMESPACE=${TOUCHSTONE_NAMESPACE:-"$network_ns"}
39+
res_output_dir="/tmp/${WORKLOAD}-${UUID}"
40+
mkdir -p ${res_output_dir}
41+
export COMPARISON_OUTPUT=${PWD}/${WORKLOAD}-${UUID}.csv
42+
final_csv=${res_output_dir}/${UUID}.csv
43+
echo "final csv $final_csv"
44+
for config in ${COMPARISON_CONFIG}
45+
do
46+
config_loc=$(dirname $(realpath ${BASH_SOURCE[0]}))/touchstone-configs/${config}
47+
echo "config ${config_loc}"
48+
check_metric_to_modify $config_loc
49+
COMPARISON_FILE="${res_output_dir}/${config}"
50+
envsubst < $config_loc > $COMPARISON_FILE
51+
echo "comparison output"
52+
if [[ -n ${ES_SERVER_BASELINE} ]] && [[ -n ${BASELINE_UUID} ]]; then
53+
log "Comparing with baseline"
54+
compare "${ES_SERVER_BASELINE} ${ES_SERVER}" "${BASELINE_UUID} ${UUID}" "${COMPARISON_FILE}" "${GEN_CSV}"
55+
else
56+
log "Querying results"
57+
compare ${ES_SERVER} ${UUID} "${COMPARISON_FILE}" "${GEN_CSV}"
58+
fi
59+
log "python csv modifier"
60+
python $(dirname $(realpath ${BASH_SOURCE[0]}))/csv_modifier.py -c ${COMPARISON_OUTPUT} -o ${final_csv}
61+
done
62+
if [[ -n ${GSHEET_KEY_LOCATION} ]] && [[ -n ${COMPARISON_OUTPUT} ]]; then
63+
echo "sheet $GSHEET_KEY_LOCATION "
64+
gen_spreadsheet ${WORKLOAD} ${final_csv} ${EMAIL_ID_FOR_RESULTS_SHEET} ${GSHEET_KEY_LOCATION}
65+
fi
66+
log "Removing touchstone"
67+
remove_touchstone
68+
fi
69+
}
70+
571

672
install_touchstone() {
773
touchstone_tmp=$(mktemp -d)
@@ -43,4 +109,4 @@ compare() {
43109
fi
44110
log "Running: ${cmd}"
45111
${cmd}
46-
}
112+
}

‎utils/csv_modifier.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import pandas as pd
22
from typing import *
3-
import sys
43
import argparse
54
import os
65

@@ -38,7 +37,10 @@ def read_csv(filepath: str) -> pd.DataFrame:
3837

3938
def divide_col(x: float) -> float:
4039
div_by = os.environ.get("div_by", 1)
41-
return round(x / float((div_by)), 3)
40+
try:
41+
return round(x / float((div_by)), 3)
42+
except:
43+
return x
4244

4345

4446
def write_df(df: pd.DataFrame, target_filepath: str, mode: str = "a"):
@@ -57,7 +59,8 @@ def main():
5759
select_columns = -1
5860
if "ES_SERVER_BASELINE" in os.environ and "BASELINE_UUID" in os.environ:
5961
select_columns = -2
60-
df.sort_values(df.columns[-1], ascending=False, inplace=True)
62+
if ("SORT_BY_VALUE" in os.environ) and (os.getenv("SORT_BY_VALUE") == "true"):
63+
df.sort_values(df.columns[-1], ascending=False, inplace=True)
6164
df[df.columns[select_columns:num_columns]] = df[
6265
df.columns[select_columns:num_columns]
6366
].apply(divide_col)
@@ -67,4 +70,4 @@ def main():
6770
)
6871

6972

70-
main()
73+
main()

‎workloads/kube-burner/touchstone-configs/clusterVersion.json renamed to ‎utils/touchstone-configs/clusterVersion.json

File renamed without changes.

‎workloads/kube-burner/touchstone-configs/containerMetrics.json renamed to ‎utils/touchstone-configs/containerMetrics.json

File renamed without changes.

‎workloads/router-perf-v2/mb-touchstone.json renamed to ‎utils/touchstone-configs/mb-touchstone.json

File renamed without changes.

‎workloads/kube-burner/touchstone-configs/podCPU-avg.json renamed to ‎utils/touchstone-configs/podCPU-avg.json

File renamed without changes.

‎workloads/kube-burner/touchstone-configs/podCPU-max.json renamed to ‎utils/touchstone-configs/podCPU-max.json

File renamed without changes.

‎workloads/kube-burner/touchstone-configs/podLatency.json renamed to ‎utils/touchstone-configs/podLatency.json

File renamed without changes.

‎workloads/kube-burner/touchstone-configs/podMemory-avg.json renamed to ‎utils/touchstone-configs/podMemory-avg.json

File renamed without changes.

‎workloads/kube-burner/touchstone-configs/podMemory-max.json renamed to ‎utils/touchstone-configs/podMemory-max.json

File renamed without changes.

‎utils/touchstone-configs/uperf-touchstone-norm-ltcy.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"elasticsearch": {
3+
"ripsaw-uperf-results": [
4+
{
5+
"filter": {
6+
"test_type.keyword": "rr"
7+
},
8+
"exclude": {
9+
"norm_ops": 0
10+
},
11+
"buckets": [
12+
"protocol.keyword",
13+
"message_size",
14+
"num_threads",
15+
"num_pairs.keyword"
16+
],
17+
"aggregations": {
18+
"norm_ltcy": [
19+
"avg"
20+
]
21+
}
22+
}
23+
]
24+
}
25+
}

‎utils/touchstone-configs/uperf-touchstone-norm-ops.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"elasticsearch": {
3+
"ripsaw-uperf-results": [
4+
{
5+
"filter": {
6+
"test_type.keyword": "rr"
7+
},
8+
"exclude": {
9+
"norm_ops": 0
10+
},
11+
"buckets": [
12+
"protocol.keyword",
13+
"message_size",
14+
"num_threads",
15+
"num_pairs.keyword"
16+
],
17+
"aggregations": {
18+
"norm_ops": [
19+
"avg"
20+
]
21+
}
22+
}
23+
]
24+
}
25+
}

‎utils/touchstone-configs/uperf-touchstone-rr.json

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"elasticsearch": {
3+
"ripsaw-uperf-results": [
4+
{
5+
"filter": {
6+
"test_type.keyword": "rr"
7+
},
8+
"exclude": {
9+
"norm_ops": 0
10+
},
11+
"buckets": [
12+
"protocol.keyword",
13+
"message_size",
14+
"num_threads",
15+
"num_pairs.keyword"
16+
],
17+
"aggregations": {
18+
"norm_ltcy": [
19+
{
20+
"percentiles": {
21+
"percents": [
22+
99
23+
]
24+
}
25+
}
26+
]
27+
}
28+
}
29+
]
30+
}
31+
}

‎workloads/network-perf/uperf-touchstone-smoke.json renamed to ‎utils/touchstone-configs/uperf-touchstone-smoke.json

File renamed without changes.

‎utils/touchstone-configs/uperf-touchstone-stream.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"elasticsearch": {
3+
"ripsaw-uperf-results": [
4+
{
5+
"filter": {
6+
"test_type.keyword": "stream"
7+
},
8+
"exclude": {
9+
"norm_ops": 0
10+
},
11+
"buckets": [
12+
"protocol.keyword",
13+
"message_size",
14+
"num_threads",
15+
"num_pairs.keyword"
16+
],
17+
"aggregations": {
18+
"norm_byte": [
19+
"avg"
20+
]
21+
}
22+
}
23+
]
24+
}
25+
}

‎workloads/network-perf/uperf-touchstone.json renamed to ‎utils/touchstone-configs/uperf-touchstone.json

File renamed without changes.

‎workloads/kube-burner/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ Workloads can be tweaked with the following environment variables:
5555
| **THANOS_RECEIVER_URL** | Thanos receiver url endpoint for grafana remote-write agent | |
5656
| **POD_READY_THRESHOLD** | Pod ready latency threshold (only applies node-density and pod-density workloads). [More info](https://kube-burner.readthedocs.io/en/latest/measurements/#pod-latency-thresholds) | 5000ms |
5757
| **PLATFORM_ALERTS** | Platform alerting enables, kube-burner alerting based cluster's platform, either ALERT_PROFILE or this variable can be set | false |
58-
| **COMPARISON_CONFIG** | Touchstone configs. Multiple config files can be passed here. Ex. COMPARISON_CONFIG="podCPU-avg.json clusterVersion.json". [Sample files](https://github.com/cloud-bulldozer/e2e-benchmarking/tree/master/workloads/kube-burner/touchstone-configs) | |
58+
| **COMPARISON_CONFIG** | Touchstone configs. Multiple config files can be passed here. Ex. COMPARISON_CONFIG="podCPU-avg.json clusterVersion.json". [Sample files](../../utils/touchstone-configs) | |
5959
| **TOUCHSTONE_NAMESPACE** | Namespace where we query for metrics specified in touchstone config files | openshift-sdn or openshift-ovn-kubernetes |
6060
| **GSHEET_KEY_LOCATION** | Location of service account key to generate google sheets | |
6161
| **EMAIL_ID_FOR_RESULTS_SHEET** | Email id where the google sheets needs to be sent | |

‎workloads/kube-burner/common.sh

Lines changed: 1 addition & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
set -m
33
source ../../utils/common.sh
44
source env.sh
5-
source ../../utils/compare.sh
5+
66

77
openshift_login
88

@@ -132,58 +132,6 @@ delete_oldpprof_folder() {
132132
rm -rf pprof-data
133133
}
134134

135-
get_network_type() {
136-
if [[ $NETWORK_TYPE == "OVNKubernetes" ]]; then
137-
network_ns=openshift-ovn-kubernetes
138-
else
139-
network_ns=openshift-sdn
140-
fi
141-
}
142-
143-
check_metric_to_modify() {
144-
export div_by=1
145-
echo $config | grep -i memory
146-
if [[ $? == 0 ]]; then
147-
export div_by=1048576
148-
fi
149-
echo $config | grep -i latency
150-
if [[ $? == 0 ]]; then
151-
export div_by=1000
152-
fi
153-
}
154-
155-
run_benchmark_comparison() {
156-
if [[ -n ${ES_SERVER} ]] && [[ -n ${COMPARISON_CONFIG} ]]; then
157-
log "Installing touchstone"
158-
install_touchstone
159-
get_network_type
160-
export TOUCHSTONE_NAMESPACE=${TOUCHSTONE_NAMESPACE:-"$network_ns"}
161-
res_output_dir="/tmp/${WORKLOAD}-${UUID}"
162-
mkdir -p ${res_output_dir}
163-
final_csv=${res_output_dir}/${UUID}.csv
164-
for config in ${COMPARISON_CONFIG}; do
165-
check_metric_to_modify
166-
envsubst < touchstone-configs/${config} > /tmp/${config}
167-
COMPARISON_OUTPUT="${res_output_dir}/${config}"
168-
if [[ -n ${ES_SERVER_BASELINE} ]] && [[ -n ${BASELINE_UUID} ]]; then
169-
log "Comparing with baseline"
170-
compare "${ES_SERVER_BASELINE} ${ES_SERVER}" "${BASELINE_UUID} ${UUID}" "/tmp/${config}" "${GEN_CSV}"
171-
else
172-
log "Querying results"
173-
compare ${ES_SERVER} ${UUID} "/tmp/${config}" "${GEN_CSV}"
174-
fi
175-
if [[ ${GEN_CSV} == "true" ]]; then
176-
python ../../utils/csv_modifier.py -c ${COMPARISON_OUTPUT} -o ${final_csv}
177-
fi
178-
done
179-
if [[ -n ${GSHEET_KEY_LOCATION} ]] && [[ ${GEN_CSV} == "true" ]]; then
180-
gen_spreadsheet ${WORKLOAD} ${final_csv} ${EMAIL_ID_FOR_RESULTS_SHEET} ${GSHEET_KEY_LOCATION}
181-
fi
182-
log "Removing touchstone"
183-
remove_touchstone
184-
fi
185-
}
186-
187135
label_node_with_label() {
188136
colon_param=$(echo $1 | tr "=" ":" | sed 's/:/: /g')
189137
export POD_NODE_SELECTOR="{$colon_param}"

‎workloads/kube-burner/env.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,4 +82,5 @@ export PLATFORM_ALERTS=${PLATFORM_ALERTS:-false}
8282
export COMPARISON_CONFIG=${COMPARISON_CONFIG:-""}
8383
export GSHEET_KEY_LOCATION=${GSHEET_KEY_LOCATION}
8484
export EMAIL_ID_FOR_RESULTS_SHEET=${EMAIL_ID_FOR_RESULTS_SHEET}
85-
GEN_CSV=${GEN_CSV:-false}
85+
export GEN_CSV=${GEN_CSV:-false}
86+
export SORT_BY_VALUE=${SORT_BY_VALUE:-true}

‎workloads/kube-burner/run.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
. common.sh
44
. build_helper.sh
5+
. ../../utils/compare.sh
56

67
label=""
78
case ${WORKLOAD} in

‎workloads/network-perf/README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,13 @@ The environment variables below are used to configure benchmark comparison and/o
4848
| **COMPARISON_ALIASES** | Benchmark-comparison aliases (UUIDs will be replaced by these aliases | "" |
4949
| **ES_SERVER_BASELINE** | Elasticsearch endpoint used used by the baseline benchmark | https://search-perfscale-dev-chmf5l4sh66lvxbnadi4bznl3a.us-west-2.es.amazonaws.com:443 |
5050
| **BASELINE_UUID** | Baseline UUID used for comparison | "" |
51-
| **COMPARISON_CONFIG** | Benchmark-comparison config file | `${PWD}/uperf-touchstone.json` |
51+
| **COMPARISON_CONFIG** | Benchmark-comparison config file,, file needs to be in **e2e-benchmarking/utils/touchstone-config **folder | [uperf-touchstone.json](../../utils/touchstone-configs/uperf-touchstone.json)|
5252
| **COMPARISON_RC** | Benchmark-comparison return code if tolerancy check fails | 0 |
5353
| **TOLERANCY_RULES_CFG** | Tolerancy rules configuration file | uperf-tolerancy-rules.yaml |
5454
| **GSHEET_KEY_LOCATION** | Location of the Google Service Account Key, used to import a resulting csv | "" |
5555
| **EMAIL_ID_FOR_RESULTS_SHEET** | Email to push CSV results | "" |
5656
| **GEN_CSV** | Generate a benchmark-comparison csv, required to generate the spreadsheet | "false" |
57+
| **SORT_BY_VALUE** | Sort the generated CSV by value | false |
5758

5859
## Snappy integration configurations
5960

‎workloads/network-perf/common.sh

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
source env.sh
44
source ../../utils/common.sh
55
source ../../utils/benchmark-operator.sh
6-
source ../../utils/compare.sh
76

87
openshift_login
98

@@ -23,24 +22,5 @@ deploy_operator() {
2322
deploy_benchmark_operator
2423
}
2524

26-
run_benchmark_comparison() {
27-
if [[ -n ${ES_SERVER} ]]; then
28-
log "Installing touchstone"
29-
install_touchstone
30-
if [[ -n ${ES_SERVER_BASELINE} ]] && [[ -n ${BASELINE_UUID} ]]; then
31-
log "Comparing with baseline"
32-
compare "${ES_SERVER_BASELINE} ${ES_SERVER}" "${BASELINE_UUID} ${UUID}" ${COMPARISON_CONFIG} ${GEN_CSV}
33-
else
34-
log "Querying results"
35-
compare ${ES_SERVER} ${UUID} ${COMPARISON_CONFIG} ${GEN_CSV}
36-
fi
37-
if [[ -n ${GSHEET_KEY_LOCATION} ]] && [[ ${GEN_CSV} == "true" ]]; then
38-
gen_spreadsheet network-performance ${COMPARISON_OUTPUT} ${EMAIL_ID_FOR_RESULTS_SHEET} ${GSHEET_KEY_LOCATION}
39-
fi
40-
log "Removing touchstone"
41-
remove_touchstone
42-
fi
43-
}
44-
4525
export_defaults
4626
deploy_operator

0 commit comments

Comments
 (0)