update experiment script

Signed-off-by: Le Xu <[email protected]>

Author: happyandslow

benchmarks/autoscaling/plot-everything.py

@@ -56,14 +56,12 @@ def parse_experiment_output(lines):
 
     return df, base_time
 
-def get_autoscaler_name(output_dir):
+def get_autoscaler_name(output_dir, workload_type):
     autoscaling = None
     print(f"output_dir: {output_dir}")
     # Extract the last part of the path after the last slash
     filename = output_dir.split("/")[-1]
-
-    # Regular expression to match the autoscaler name
-    match = re.search(r"^[^-]+-[^-]+-([^-]+(?:-[^-]+)*)-\d{8}-\d{6}$", filename)
+    match = re.search(rf"^[^-]+-{workload_type}-([^-]+(?:-[^-]+)*)-\d{{8}}-\d{{6}}$", filename)
 
     if match:
         print(match)
@@ -246,11 +244,10 @@ def plot_combined_visualization(experiment_home_dir, workload_type):
         output_dir = os.path.join(experiment_home_dir, subdir)
         if "pod_logs" in output_dir:
             continue
-        autoscaler = get_autoscaler_name(output_dir)
+        autoscaler = get_autoscaler_name(output_dir, workload_type)
         color = colors[autoscaler]
         marker = '.'
         label_name = f'{autoscaler}'
-
         # Read and parse data
         experiment_output_file = os.path.join(output_dir, "output.jsonl")
         parsed_lines = read_experiment_file(experiment_output_file)
@@ -380,7 +377,7 @@ def plot_combined_visualization(experiment_home_dir, workload_type):
         content = read_stats_file(stat_fn)
         if content:
             stats = parse_performance_stats(content)
-            autoscaler = get_autoscaler_name(output_dir)
+            autoscaler = get_autoscaler_name(output_dir, workload_type)
             title = f"{autoscaler}"
             if autoscaler is None or autoscaler == "none" or autoscaler == "NONE":
                 color_list.append(colors[autoscaler])

benchmarks/autoscaling/requirements_bench_pa.txt

@@ -114,7 +114,7 @@ python3 ${aibrix_repo}/benchmarks/client/client.py \
     --model ${target_ai_model} \
     --api-key ${api_key} \
     --output-file-path ${output_jsonl_path} \
-    #--output-dir ${experiment_result_dir} \
+    --streaming
 
 echo "Experiment is done. date: $(date)"
 

benchmarks/autoscaling/run-test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
 set -x
-#./run.sh workload/workload/25min_up_and_down/25min_up_and_down.jsonl
 
-export KUBECONFIG=~/.kube/config-vke
-export aibrix_repo="/root/aibrix-local"
-export api_key="sk-kFJ12nKsFVfVmGpj3QzX65s4RbN2xJqWzPYCjYu7wT3BlbLi"
-export kube_context="ccr3aths9g2gqedu8asdg@35122069-kcu0n2lfb7pjdd83330h0"
+export KUBECONFIG=${KUBECONFIG}
+export aibrix_repo=${aibrix_repo}
+export api_key=${api_key}
+export kube_context=${kube_context}
 
 for WORKLOAD_TYPE in "T_HighSlow_I_HighSlow_O_HighFast"  "T_HighSlow_I_HighSlow_O_HighSlow" "T_HighSlow_I_LowFast_O_HighSlow" "T_HighSlow_I_LowSlow_O_HighSlow"
 do
@@ -16,6 +15,35 @@ do
         exit 1
     fi
 
+    autoscalers="hpa kpa apa optimizer-kpa"
+    for autoscaler in ${autoscalers}; do
+        start_time=$(date +%s)
+        echo "--------------------------------"
+        echo "started experiment at $(date)"
+        echo autoscaler: ${autoscaler}
+        echo workload: ${workload_path} 
+        echo "The stdout/stderr is being logged in output-${autoscaler}-${WORKLOAD_TYPE}.txt"
+        ./run-test.sh ${workload_path} ${autoscaler} ${aibrix_repo} ${api_key} ${kube_context} ${WORKLOAD_TYPE} > output-${autoscaler}-${WORKLOAD_TYPE}.txt  2>&1
+        end_time=$(date +%s)
+        echo "Done: Time taken: $((end_time-start_time)) seconds"
+        echo "--------------------------------"
+        sleep 10
+    done
+    python plot-everything.py experiment_results/${WORKLOAD_TYPE} ${WORKLOAD_TYPE}
+done
+
+
+
+
+for WORKLOAD_TYPE in "workload-2024-10-10-19-50-00" "workload-2024-10-15-18-50-00" 
+do
+    workload_path="workload/maas/${WORKLOAD_TYPE}/internal.jsonl"
+    if [ -z "${workload_path}" ]; then
+        echo "workload path is not given"
+        echo "Usage: $0 <workload_path>"
+        exit 1
+    fi
+
     autoscalers="hpa kpa apa optimizer-kpa"
     for autoscaler in ${autoscalers}; do
         start_time=$(date +%s)
@@ -30,9 +58,24 @@ do
         echo "--------------------------------"
         sleep 10
     done
+    python plot-everything.py experiment_results/${WORKLOAD_TYPE} ${WORKLOAD_TYPE}
 done
 
-# for WORKLOAD_TYPE in "T_HighSlow_I_HighSlow_O_HighFast"  "T_HighSlow_I_HighSlow_O_HighSlow" "T_HighSlow_I_LowFast_O_HighSlow" "T_HighSlow_I_LowSlow_O_HighSlow"
+
+
+# target_deployment="deepseek-llm-7b-chat"
+# kubectl delete podautoscaler --all --all-namespaces
+# python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1 --context ${kube_context}
+# target_ai_model=deepseek-llm-7b-chat
+
+
+# mkdir -p output-profile
+# for qps in {1..10} 
 # do
-#     python plot-everything.py experiment_results/${WORKLOAD_TYPE}
-# done
+#     kubectl -n envoy-gateway-system port-forward service/envoy-aibrix-system-aibrix-eg-903790dc 8888:80 &
+#     STRATEGY="random"
+#     WORKLOAD_PATH=workload/constant/qps-${qps}/constant.jsonl
+#     python3 ${aibrix_repo}/benchmarks/client/client.py --workload-path ${WORKLOAD_PATH} --endpoint "http://localhost:8888" --model ${target_ai_model}  --api-key ${api_key} --output-file-path output-profile/output-qps${qps}.jsonl
+#     # python analyze.py output-profile/output-qps${qps}.jsonl
+#     sleep 30
+# done