Autoscaling Benchmark Initial

benchmarks/autoscaling/README.md

@@ -43,10 +43,10 @@ For example,
 There are two plots that you can plot.
 
 ### Generating report
-`python <aibrix_root_repo>/benchmarks/plot/plot-everything.py <experiment_home_dir>`
+`python plot-everything.py <experiment_home_dir>`
 
 For example, 
-`python <aibrix_root_repo>/benchmarks/plot/plot-everything.py experiment_results/25min_test`
+`python plot-everything.py experiment_results/25min_test`
 
 The directories should look like
 ```bash

benchmarks/autoscaling/deepseek-llm-7b-chat/deploy.yaml

@@ -176,5 +176,5 @@ spec:
                   - key: machine.cluster.vke.volcengine.com/gpu-name
                     operator: In
                     values:
-                      # - NVIDIA-L20
-                      - Tesla-V100
+                      - NVIDIA-L20
+                      # - Tesla-V100

benchmarks/autoscaling/plot-everything.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import os
 import json
+import re
 import pandas as pd
 import matplotlib.pyplot as plt
 import numpy as np
@@ -24,10 +25,15 @@ def parse_experiment_output(lines):
             continue
         try:
             data = json.loads(line.strip())
-            required_fields = ['status_code', 'start_time', 'end_time', 'latency', 'throughput', 
+            # required_fields = ['status_code', 'start_time', 'end_time', 'latency', 'throughput', 
+            #                  'prompt_tokens', 'output_tokens', 'total_tokens', 'input', 'output']
+            required_fields = ['status', 'start_time', 'end_time', 'latency', 'throughput', 
                              'prompt_tokens', 'output_tokens', 'total_tokens', 'input', 'output']
             if any(field not in data for field in required_fields):
+                missingfields = [field not in data for field in required_fields]
+                print(missingfields)
                 continue
+            data['status_code'] = 200
             results.append(data)
         except json.JSONDecodeError:
             continue
@@ -43,24 +49,34 @@ def parse_experiment_output(lines):
     rps_series = df.groupby('second_bucket').size()
     df['rps'] = df['second_bucket'].map(rps_series)
 
-    success_rps = df[df['status_code'] == 200].groupby('second_bucket').size()
-    failed_rps = df[df['status_code'] != 200].groupby('second_bucket').size()
+    success_rps = df[df['status'] == 'success'].groupby('second_bucket').size()
+    failed_rps = df[df['status'] != 'success'].groupby('second_bucket').size()
     df['success_rps'] = df['second_bucket'].map(success_rps).fillna(0)
     df['failed_rps'] = df['second_bucket'].map(failed_rps).fillna(0)
 
     return df, base_time
 
-def get_autoscaler_name(output_dir):
+def get_autoscaler_name(output_dir, workload_type):
     autoscaling = None
-    with open(f"{output_dir}/output.txt", 'r', encoding='utf-8') as f_:
-        lines = f_.readlines()
-        for line in lines:
-            if "autoscaler" in line:
-                autoscaling = line.split(":")[-1].strip()
-                break
+    print(f"output_dir: {output_dir}")
+    # Extract the last part of the path after the last slash
+    filename = output_dir.split("/")[-1]
+    match = re.search(rf"^[^-]+-{workload_type}-([^-]+(?:-[^-]+)*)-\d{{8}}-\d{{6}}$", filename)
+
+    if match:
+        print(match)
+        autoscaling = match.group(1)
+
+    # with open(f"{output_dir}/output.txt", 'r', encoding='utf-8') as f_:
+    #     lines = f_.readlines()
+    #     for line in lines:
+    #         if "autoscaler" in line:
+    #             autoscaling = line.split(":")[-1].strip()
+    #             break
     if autoscaling == None:
         print(f"Invalid parsed autoscaling name: {autoscaling}")
         assert False
+    print(autoscaling)
     return autoscaling.upper()
 
 def parse_performance_stats(file_content):
@@ -160,7 +176,7 @@ def analyze_performance(df):
         raise Exception(f"Error analyzing performance metrics: {e}")
 
 
-def plot_combined_visualization(experiment_home_dir):
+def plot_combined_visualization(experiment_home_dir, workload_type):
     # Create figure
     fig = plt.figure(figsize=(12, 12))
 
@@ -228,11 +244,10 @@ def plot_combined_visualization(experiment_home_dir):
         output_dir = os.path.join(experiment_home_dir, subdir)
         if "pod_logs" in output_dir:
             continue
-        autoscaler = get_autoscaler_name(output_dir)
+        autoscaler = get_autoscaler_name(output_dir, workload_type)
         color = colors[autoscaler]
         marker = '.'
         label_name = f'{autoscaler}'
-
         # Read and parse data
         experiment_output_file = os.path.join(output_dir, "output.jsonl")
         parsed_lines = read_experiment_file(experiment_output_file)
@@ -362,7 +377,7 @@ def plot_combined_visualization(experiment_home_dir):
         content = read_stats_file(stat_fn)
         if content:
             stats = parse_performance_stats(content)
-            autoscaler = get_autoscaler_name(output_dir)
+            autoscaler = get_autoscaler_name(output_dir, workload_type)
             title = f"{autoscaler}"
             if autoscaler is None or autoscaler == "none" or autoscaler == "NONE":
                 color_list.append(colors[autoscaler])
@@ -400,18 +415,19 @@ def plot_combined_visualization(experiment_home_dir):
     plt.tight_layout()
 
     # Save the combined figure
-    output_path = os.path.join(experiment_home_dir, 'combined_visualization.pdf')
+    output_path = os.path.join(experiment_home_dir, f'combined_visualization-{workload_type}.pdf')
     plt.savefig(output_path, bbox_inches='tight')
     print(f"** Saved combined visualization to: {output_path}")
     plt.close()
 
 def main():
-    if len(sys.argv) != 2:
-        print("Usage: python script.py <experiment_home_dir>")
+    if len(sys.argv) != 3:
+        print("Usage: python script.py <experiment_home_dir> <workload_type>")
         return 1
 
     experiment_home_dir = sys.argv[1]
-    plot_combined_visualization(experiment_home_dir)
+    workload_type = sys.argv[2]
+    plot_combined_visualization(experiment_home_dir, workload_type)
     return 0
 
 if __name__ == "__main__":

benchmarks/autoscaling/run-test.sh

@@ -2,8 +2,11 @@
 
 input_workload_path=$1
 autoscaler=$2
-aibrix_repo="" # root dir of aibrix repo
-api_key="" # set your api key
+aibrix_repo=$3 # root dir of aibrix repo
+api_key=$4 # set your api key
+kube_context=$5
+workload_type=$6
+
 k8s_yaml_dir="deepseek-llm-7b-chat"
 target_deployment="deepseek-llm-7b-chat" # "aibrix-model-deepseek-llm-7b-chat"
 target_ai_model=deepseek-llm-7b-chat
@@ -37,7 +40,7 @@ fi
 
 # Setup experiment directory
 workload_name=$(echo $input_workload_path | tr '/' '\n' | grep .jsonl | cut -d '.' -f 1)
-experiment_result_dir="experiment_results/${workload_name}-${autoscaler}-$(date +%Y%m%d-%H%M%S)"
+experiment_result_dir="experiment_results/${workload_type}/${workload_name}-${workload_type}-${autoscaler}-$(date +%Y%m%d-%H%M%S)"
 if [ ! -d ${experiment_result_dir} ]; then
     echo "output directory does not exist. Create the output directory (${experiment_result_dir})"
     mkdir -p ${experiment_result_dir}
@@ -66,7 +69,7 @@ kubectl delete -f ${k8s_yaml_dir}/deploy.yaml
 kubectl apply -f ${k8s_yaml_dir}/deploy.yaml
 kubectl apply -f ${k8s_yaml_dir}/${autoscaler}.yaml
 echo "kubectl apply -f ${k8s_yaml_dir}/${autoscaler}.yaml"
-python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1
+python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1 --context ${kube_context}
 echo "Set number of replicas to \"1\". Autoscaling experiment will start from 1 pod"
 
 echo "Restart aibrix-controller-manager deployment"
@@ -82,9 +85,9 @@ kubectl rollout restart deploy ${target_deployment} -n default
 sleep_before_pod_check=20
 echo "Sleep for ${sleep_before_pod_check} seconds after restarting deployment"
 sleep ${sleep_before_pod_check}
-python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py ${target_deployment}
-python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-controller-manager
-python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-gateway-plugins
+python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py ${target_deployment} ${kube_context}
+python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-controller-manager ${kube_context}
+python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-gateway-plugins ${kube_context}
 
 # Start pod log monitoring
 pod_log_dir="${experiment_result_dir}/pod_logs"
@@ -94,7 +97,7 @@ mkdir -p ${pod_log_dir}
 cp ${input_workload_path} ${experiment_result_dir}
 
 # Start pod counter. It will run on background until the end of the experiment.
-python3 ${aibrix_repo}/benchmarks/utils/count_num_pods.py ${target_deployment} ${experiment_result_dir} &
+python3 ${aibrix_repo}/benchmarks/utils/count_num_pods.py ${target_deployment} ${experiment_result_dir} ${kube_context} &
 COUNT_NUM_POD_PID=$!
 echo "started count_num_pods.py with PID: $COUNT_NUM_POD_PID"
 
@@ -105,13 +108,13 @@ python3 ${aibrix_repo}/benchmarks/utils/streaming_pod_log_to_file.py aibrix-gate
 
 # Run experiment!!!
 output_jsonl_path=${experiment_result_dir}/output.jsonl
-python3 ${aibrix_repo}/benchmarks/generator/client.py \
+python3 ${aibrix_repo}/benchmarks/client/client.py \
     --workload-path ${input_workload_path} \
-    --endpoint "localhost:8888" \
+    --endpoint "http://localhost:8888" \
     --model ${target_ai_model} \
     --api-key ${api_key} \
-    --output-dir ${experiment_result_dir} \
-    --output-file-path ${output_jsonl_path}
+    --output-file-path ${output_jsonl_path} \
+    --streaming
 
 echo "Experiment is done. date: $(date)"
 
@@ -123,7 +126,7 @@ sleep 1
 
 # Cleanup
 kubectl delete podautoscaler --all --all-namespaces
-python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1
+python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1 --context ${kube_context}
 kubectl delete -f ${k8s_yaml_dir}/deploy.yaml
 
 # Stop monitoring processes

benchmarks/autoscaling/run.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -x
+
+export KUBECONFIG=${KUBECONFIG}
+export aibrix_repo=${aibrix_repo}
+export api_key=${api_key}
+export kube_context=${kube_context}
+
+for WORKLOAD_TYPE in "T_HighSlow_I_HighSlow_O_HighFast"  "T_HighSlow_I_HighSlow_O_HighSlow" "T_HighSlow_I_LowFast_O_HighSlow" "T_HighSlow_I_LowSlow_O_HighSlow"
+do
+    workload_path="workload/synthetic_patterns/${WORKLOAD_TYPE}/synthetic_manual_config.jsonl"
+    if [ -z "${workload_path}" ]; then
+        echo "workload path is not given"
+        echo "Usage: $0 <workload_path>"
+        exit 1
+    fi
+
+    autoscalers="hpa kpa apa optimizer-kpa"
+    for autoscaler in ${autoscalers}; do
+        start_time=$(date +%s)
+        echo "--------------------------------"
+        echo "started experiment at $(date)"
+        echo autoscaler: ${autoscaler}
+        echo workload: ${workload_path} 
+        echo "The stdout/stderr is being logged in output-${autoscaler}-${WORKLOAD_TYPE}.txt"
+        ./run-test.sh ${workload_path} ${autoscaler} ${aibrix_repo} ${api_key} ${kube_context} ${WORKLOAD_TYPE} > output-${autoscaler}-${WORKLOAD_TYPE}.txt  2>&1
+        end_time=$(date +%s)
+        echo "Done: Time taken: $((end_time-start_time)) seconds"
+        echo "--------------------------------"
+        sleep 10
+    done
+    python plot-everything.py experiment_results/${WORKLOAD_TYPE} ${WORKLOAD_TYPE}
+done
+
+
+
+
+for WORKLOAD_TYPE in "workload-2024-10-10-19-50-00" "workload-2024-10-15-18-50-00" 
+do
+    workload_path="workload/maas/${WORKLOAD_TYPE}/internal.jsonl"
+    if [ -z "${workload_path}" ]; then
+        echo "workload path is not given"
+        echo "Usage: $0 <workload_path>"
+        exit 1
+    fi
+
+    autoscalers="hpa kpa apa optimizer-kpa"
+    for autoscaler in ${autoscalers}; do
+        start_time=$(date +%s)
+        echo "--------------------------------"
+        echo "started experiment at $(date)"
+        echo autoscaler: ${autoscaler}
+        echo workload: ${workload_path} 
+        echo "The stdout/stderr is being logged in output-${WORKLOAD_TYPE}.txt"
+        ./run-test.sh ${workload_path} ${autoscaler} ${aibrix_repo} ${api_key} ${kube_context} ${WORKLOAD_TYPE} > output-${WORKLOAD_TYPE}.txt  2>&1
+        end_time=$(date +%s)
+        echo "Done: Time taken: $((end_time-start_time)) seconds"
+        echo "--------------------------------"
+        sleep 10
+    done
+    python plot-everything.py experiment_results/${WORKLOAD_TYPE} ${WORKLOAD_TYPE}
+done
+
+
+
+# target_deployment="deepseek-llm-7b-chat"
+# kubectl delete podautoscaler --all --all-namespaces
+# python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1 --context ${kube_context}
+# target_ai_model=deepseek-llm-7b-chat
+
+
+# mkdir -p output-profile
+# for qps in {1..10} 
+# do
+#     kubectl -n envoy-gateway-system port-forward service/envoy-aibrix-system-aibrix-eg-903790dc 8888:80 &
+#     STRATEGY="random"
+#     WORKLOAD_PATH=workload/constant/qps-${qps}/constant.jsonl
+#     python3 ${aibrix_repo}/benchmarks/client/client.py --workload-path ${WORKLOAD_PATH} --endpoint "http://localhost:8888" --model ${target_ai_model}  --api-key ${api_key} --output-file-path output-profile/output-qps${qps}.jsonl
+#     # python analyze.py output-profile/output-qps${qps}.jsonl
+#     sleep 30
+# done