Merge pull request #29273 from taosdata/cover/3.0

merge: from cover/3.0 to 3.0 branch
taosdata · Dec 23, 2024 · eafbc5e · eafbc5e
2 parents 81fc62b + 164cd03
commit eafbc5e
Show file tree

Hide file tree

Showing 6 changed files with 260 additions and 46 deletions.
diff --git a/tests/pytest/auto_crash_gen.py b/tests/pytest/auto_crash_gen.py
@@ -384,7 +384,8 @@ def main():
         if text_result == "success":
             send_msg(notification_robot_url, get_msg(text))
         else:
-            send_msg(alert_robot_url, get_msg(text))    
+            send_msg(alert_robot_url, get_msg(text))   
+            send_msg(notification_robot_url, get_msg(text)) 
 
         #send_msg(get_msg(text)) 
     except Exception as e:

diff --git a/tests/pytest/auto_crash_gen_valgrind.py b/tests/pytest/auto_crash_gen_valgrind.py
@@ -419,6 +419,7 @@ def main():
             send_msg(notification_robot_url, get_msg(text))
         else:
             send_msg(alert_robot_url, get_msg(text))    
+            send_msg(notification_robot_url, get_msg(text))
 
         #send_msg(get_msg(text)) 
     except Exception as e:

diff --git a/tests/pytest/auto_crash_gen_valgrind_cluster.py b/tests/pytest/auto_crash_gen_valgrind_cluster.py
@@ -406,7 +406,8 @@ def main():
         if text_result == "success":
             send_msg(notification_robot_url, get_msg(text))
         else:
-            send_msg(alert_robot_url, get_msg(text))    
+            send_msg(alert_robot_url, get_msg(text))  
+            send_msg(notification_robot_url, get_msg(text))  
 
         #send_msg(get_msg(text)) 
     except Exception as e:

diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh
@@ -7,36 +7,171 @@ GREEN_DARK='\033[0;32m'
 GREEN_UNDERLINE='\033[4;32m'
 NC='\033[0m'
 
-TDENGINE_DIR=/root/TDinternal/community
-
+function print_color() {
+    local color="$1"
+    local message="$2"
+    echo -e "${color}${message}${NC}"
+}
 
-#echo "TDENGINE_DIR = $TDENGINE_DIR"
+# 初始化参数
+TDENGINE_DIR="/root/TDinternal/community"
+BRANCH=""
+SAVE_LOG="notsave"
+
+# 解析命令行参数
+while getopts "hd:b:t:s:" arg; do
+  case $arg in
+    d)
+      TDENGINE_DIR=$OPTARG
+      ;;
+    b)
+      BRANCH=$OPTARG
+      ;;
+    s)
+      SAVE_LOG=$OPTARG
+      ;;
+    h)
+      echo "Usage: $(basename $0) -d [TDengine_dir] -b [branch] -s [save ci case log]"
+      echo "                  -d [TDengine_dir] [default /root/TDinternal/community] "
+      echo "                  -b [branch] [default local branch] "
+      echo "                  -s [save/notsave] [default save ci case log in TDengine_dir/tests/ci_bak] "
+      exit 0
+      ;;
+    ?)
+      echo "Usage: ./$(basename $0) -h"
+      exit 1
+      ;;
+  esac
+done
+
+# 检查是否提供了命令名称
+if [ -z "$TDENGINE_DIR" ]; then
+  echo "Error: TDengine dir is required."
+  echo "Usage: $(basename $0) -d [TDengine_dir] -b [branch] -s [save ci case log] "
+  echo "                        -d [TDengine_dir] [default /root/TDinternal/community] "
+  echo "                        -b [branch] [default local branch] "
+  echo "                        -s [save/notsave] [default save ci case log in TDengine_dir/tests/ci_bak] "    
+  exit 1
+fi
+
+
+echo "TDENGINE_DIR = $TDENGINE_DIR"
 today=`date +"%Y%m%d"`
-TDENGINE_ALLCI_REPORT=$TDENGINE_DIR/tests/all-ci-report-$today.log
+TDENGINE_ALLCI_REPORT="$TDENGINE_DIR/tests/all-ci-report-$today.log"
+BACKUP_DIR="$TDENGINE_DIR/tests/ci_bak"
+mkdir -p "$BACKUP_DIR"
+#cd $BACKUP_DIR && rm -rf *
+
+
+function buildTDengine() {
+    print_color "$GREEN" "TDengine build start"
+
+    # pull parent code
+    cd "$TDENGINE_DIR/../"
+    print_color "$GREEN" "git pull parent code..."
+    git remote prune origin > /dev/null
+    git remote update > /dev/null
+
+    # pull tdengine code
+    cd $TDENGINE_DIR
+    print_color "$GREEN" "git pull tdengine code..."
+    git remote prune origin > /dev/null
+    git remote update > /dev/null
+    REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch`
+    LOCAL_COMMIT=`git rev-parse --short @`
+    print_color "$GREEN" " LOCAL: $LOCAL_COMMIT"
+    print_color "$GREEN" "REMOTE: $REMOTE_COMMIT"
+
+    if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+        print_color "$GREEN" "repo up-to-date"
+    else
+        print_color "$GREEN" "repo need to pull"
+    fi
+
+    git reset --hard
+    git checkout -- .
+    git checkout $branch
+    git checkout -- .
+    git clean -f
+    git pull
+
+    [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
+    cd $TDENGINE_DIR/debug
+
+    print_color "$GREEN" "rebuild.."
+    LOCAL_COMMIT=`git rev-parse --short @`
+
+    rm -rf *
+    makecmd="cmake -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../"
+    print_color "$GREEN" "$makecmd"
+    $makecmd
+
+    make -j 8 install
+
+    print_color "$GREEN" "TDengine build end"
+}
+
+
+# 检查并获取分支名称
+if [ -n "$BRANCH" ]; then
+    branch="$BRANCH"
+    print_color "$GREEN" "Testing branch: $branch "
+    print_color "$GREEN" "Build is required for this test！"
+    buildTDengine
+else
+    print_color "$GREEN" "Build is not required for this test！"
+fi
 
 
 function runCasesOneByOne () {
     while read -r line; do
         if [[ "$line" != "#"* ]]; then
             cmd=`echo $line | cut -d',' -f 5`
             if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then
+                echo $cmd
                 case=`echo $cmd | cut -d' ' -f 3`
+                case_file=`echo $case | tr -d ' /' `
                 start_time=`date +%s`
-                date +%F\ %T | tee -a  $TDENGINE_ALLCI_REPORT  && timeout 20m $cmd > /dev/null 2>&1 && \
-                echo -e "${GREEN}$case success${NC}" | tee -a  $TDENGINE_ALLCI_REPORT \
-                || echo -e "${RED}$case failed${NC}" | tee -a  $TDENGINE_ALLCI_REPORT
+                date +%F\ %T | tee -a  $TDENGINE_ALLCI_REPORT  && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \
+                echo -e "${GREEN}$case success${NC}" | tee -a  $TDENGINE_ALLCI_REPORT || \
+                echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT
+
+                # # 记录日志和备份
+                # mkdir -p "$BACKUP_DIR/$case_file"
+                # tar --exclude='*.sock*' -czf "$BACKUP_DIR/$case_file/sim.tar.gz" -C "$TDENGINE_DIR/.." sim
+                # mv "$TDENGINE_DIR/tests/$case_file.log" "$BACKUP_DIR/$case_file"
+
+                if [ "$SAVE_LOG" == "save" ]; then
+                    mkdir -p "$BACKUP_DIR/$case_file"
+                    tar --exclude='*.sock*' -czf "$BACKUP_DIR/$case_file/sim.tar.gz" -C "$TDENGINE_DIR/.." sim
+                    mv "$TDENGINE_DIR/tests/$case_file.log" "$BACKUP_DIR/$case_file"
+                else
+                    echo "This case not save log!"
+                fi
+
                 end_time=`date +%s`
                 echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT
-                
+
             elif [[ "$line" == *"$2"* ]]; then
+                echo $cmd
                 if [[ "$cmd" == *"pytest.sh"* ]]; then
                     cmd=`echo $cmd | cut -d' ' -f 2-20`
                 fi
-                case=`echo $cmd | cut -d' ' -f 4-20`
+                case=`echo $cmd | cut -d' ' -f 4-20`           
+                case_file=`echo $case | tr -d ' /' `
                 start_time=`date +%s`
-                date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > /dev/null 2>&1 && \
+                date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \
                 echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \
                 echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT
+
+                if [ "$SAVE_LOG" == "save" ]; then
+                    mkdir -p "$BACKUP_DIR/$case_file"
+                    tar --exclude='*.sock*' -czf "$BACKUP_DIR/$case_file/sim.tar.gz" -C "$TDENGINE_DIR/.." sim
+                    mv "$TDENGINE_DIR/tests/$case_file.log" "$BACKUP_DIR/$case_file"
+                else
+                    echo "This case not save log!"
+                fi
+
                 end_time=`date +%s`
                 echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT
             fi
@@ -45,62 +180,62 @@ function runCasesOneByOne () {
 }
 
 function runUnitTest() {
-    echo "=== Run unit test case ==="
-    echo " $TDENGINE_DIR/debug"
-    cd $TDENGINE_DIR/debug
+    print_color "$GREEN" "=== Run unit test case ==="
+    print_color "$GREEN" " $TDENGINE_DIR/../debug"
+    cd $TDENGINE_DIR/../debug
     ctest -j12
-    echo "3.0 unit test done"
+    print_color "$GREEN" "3.0 unit test done"
 }
 
 function runSimCases() {
-    echo "=== Run sim cases ==="
+    print_color "$GREEN" "=== Run sim cases ==="
 
     cd $TDENGINE_DIR/tests/script
-    runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/cases-test.task sim
+    runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/cases.task sim
 
     totalSuccess=`grep 'sim success' $TDENGINE_ALLCI_REPORT | wc -l`
     if [ "$totalSuccess" -gt "0" ]; then
-        echo "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT
+        print_color "$GREEN" "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT
     fi
 
     totalFailed=`grep 'sim failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l`
     if [ "$totalFailed" -ne "0" ]; then
-        echo "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT
+        print_color "$RED" "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT
     fi
 }
 
 function runPythonCases() {
-    echo "=== Run python cases ==="
+    print_color "$GREEN" "=== Run python cases ==="
 
     cd $TDENGINE_DIR/tests/parallel_test
-    sed -i '/compatibility.py/d' cases-test.task
+    sed -i '/compatibility.py/d' cases.task
 
     # army
     cd $TDENGINE_DIR/tests/army
-    runCasesOneByOne ../parallel_test/cases-test.task army
+    runCasesOneByOne ../parallel_test/cases.task army
 
     # system-test
     cd $TDENGINE_DIR/tests/system-test
-    runCasesOneByOne ../parallel_test/cases-test.task system-test
+    runCasesOneByOne ../parallel_test/cases.task system-test
 
     # develop-test
     cd $TDENGINE_DIR/tests/develop-test
-    runCasesOneByOne ../parallel_test/cases-test.task develop-test
+    runCasesOneByOne ../parallel_test/cases.task develop-test
 
     totalSuccess=`grep 'py success' $TDENGINE_ALLCI_REPORT | wc -l`
     if [ "$totalSuccess" -gt "0" ]; then
-        echo "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT
+        print_color "$GREEN" "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT
     fi
 
     totalFailed=`grep 'py failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l`
     if [ "$totalFailed" -ne "0" ]; then
-        echo "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT
+        print_color "$RED" "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT
     fi
 }
 
 
 function runTest() {
-    echo "run Test"
+    print_color "$GREEN" "run Test"
 
     cd $TDENGINE_DIR
     [ -d sim ] && rm -rf sim
@@ -119,20 +254,20 @@ function runTest() {
 }
 
 function stopTaosd {
-    echo "Stop taosd start"
-        systemctl stop taosd
-      PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+    print_color "$GREEN" "Stop taosd start"
+    systemctl stop taosd
+    PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
     while [ -n "$PID" ]
     do
-    pkill -TERM -x taosd
-    sleep 1
-      PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+        pkill -TERM -x taosd
+        sleep 1
+        PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
     done
-    echo "Stop tasod end"
+    print_color "$GREEN" "Stop tasod end"
 }
 
 function stopTaosadapter {
-    echo "Stop taosadapter"
+    print_color "$GREEN" "Stop taosadapter"
     systemctl stop taosadapter.service
     PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'`
     while [ -n "$PID" ]
@@ -141,18 +276,18 @@ function stopTaosadapter {
         sleep 1
         PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
     done
-    echo "Stop tasoadapter end"
+    print_color "$GREEN" "Stop tasoadapter end"
 
 }
 
 WORK_DIR=/root/
 
 date >> $WORK_DIR/date.log
-echo "Run ALL CI Test Cases" | tee -a $WORK_DIR/date.log
+print_color "$GREEN" "Run all ci test cases" | tee -a $WORK_DIR/date.log
 
 stopTaosd
 
 runTest
 
 date >> $WORK_DIR/date.log
-echo "End of CI Test Cases" | tee -a $WORK_DIR/date.log
+print_color "$GREEN" "End of ci test cases" | tee -a $WORK_DIR/date.log