DB-11134 start-splice-cluster to abort when start fails, added pipelines/upgrade-testing

- check_upgrade.sh {VERSION} {additional start-splice-cluster parameters}"
- create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters}

Author: martinrupp

check_upgrade.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+if [ $# -lt 2 ]
+then
+	echo "usage: bash create_upgrade_targz.sh {VERSION} {additional start-splice-cluster parameters}"
+	echo "------------------------------------------------------------------------------------------"
+	echo "uses a previously created tar.gz to test upgrade"
+	echo "e.g. bash create_upgrade_targz.sh 3.2.2021 -T 16"
+	echo "make sure you current branch has already been build"
+	echo "don't use -b, since we are deleting some files in platform_it/target"
+    
+    exit 1
+fi
+
+VERSION=${1}
+shift
+
+# stop current cluster
+./start-splice-cluster -k
+
+# clean up platform_it
+cd platform_it
+git clean -dfx
+cd ..
+
+# download the previous standalone data
+aws s3 cp ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz .
+tar -xzvf platform_it_${VERSION}.tar.gz
+rm platform_it_${VERSION}.tar.gz
+
+# restart cluster
+./start-splice-cluster -l $*
+
+# test
+if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then
+    echo "UPGRADE SUCCEEDED"
+    cat platform_it/splice.log | grep 'upgrade scripts'
+    cat platform_it/splice.log | grep 'Running upgrade script'	
+else
+	echo "!!! UPGRADE FAILED !!!"
+fi

pipelines/upgrade-testing/check_upgrade.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+
+if [ $# -lt 2 ]
+then
+    echo "usage: bash platform_it/src/test/bin/create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters}"
+    echo "----------------------------------------------------------------------------------------------------------------------------------"
+    echo "creates a tar.gz of a spliceengine standalone cluster that can be used to test upgrade"
+    echo "e.g. bash create_upgrade_targz.sh 3.1.0.1971 test -T 16 -pcore,cdh6.3.0"
+    exit 1
+fi
+
+VERSION=${1} # e.g. 3.1.0.1971
+MODE=${2} # test or upload
+shift 2
+
+
+PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD`
+git stash
+
+# creates a file platform_it_${VERSION}.tar.gz
+git checkout tags/${VERSION}
+cd platform_it
+rm -rf target *.log snappy*.jnilib
+cd ..
+
+./start-splice-cluster $*
+
+rm -rf upgrade_test_TMP
+mkdir -p upgrade_test_TMP/platform_it/target
+cd upgrade_test_TMP
+cp -r ../platform_it/target/hbase platform_it/target/.
+cp -r ../platform_it/target/zookeeper platform_it/target/.
+tar -czvf ../platform_it_${VERSION}.tar.gz platform_it
+cd ..
+rm -rf upgrade_test_TMP
+
+if [[ $MODE == "upload" ]]; then
+    aws s3 cp platform_it_${VERSION}.tar.gz ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz
+fi
+
+git checkout ${PREVIOUS_BRANCH}
+git stash pop

pipelines/upgrade-testing/create_upgrade_targz.sh

@@ -64,18 +64,24 @@ function is_port_open
 }
 
 ##################################################################################
-# print message ${1} and wait until port {2} is open
+# print message ${1} and wait until port {2} is open, for max {3}*2 seconds
 ##################################################################################
 function _wait_for_port
 {
   msg=${1}
   port=${2}
+  wait_max=${3}
   echo -n ${msg}
   echo -n " "
   counter=0
   until is_port_open localhost ${port}; do
     echo -n "${counter} - "
     counter=$((counter+1))
+
+    if [ ${counter} -ge ${wait_max} ]; then
+       echo "Waited too long, aborting."
+       exit 2
+    fi    
     sleep 2
   done
   echo "done!"
@@ -175,7 +181,7 @@ function _start_region_servers {
 function _wait_for_region_servers {
   if [[ ${MEMBERS} -gt 0 ]]; then
     for (( MEMBER=1; MEMBER<${MEMBERS}; MEMBER++ )); do
-      _wait_for_port "Waiting for Region Server $(($MEMBER +1)) to be ready ..." $(( 1527 + ${MEMBER} ))
+      _wait_for_port "Waiting for Region Server $(($MEMBER +1)) to be ready ..." $(( 1527 + ${MEMBER} )) 60
     done
   fi
 }
@@ -198,7 +204,27 @@ function _start_master {
 }
 
 function _wait_for_master {
-  _wait_for_port "Waiting for Master to be ready ..." 1527
+
+  msg="Waiting for Master to be ready ..."
+  port=1527
+  wait_max=60
+  echo -n ${msg}
+  echo -n " "
+  counter=0
+  until is_port_open localhost ${port}; do
+    echo -n "${counter} - "
+    counter=$((counter+1))
+    if [ ${counter} -ge ${wait_max} ]; then
+       echo "Waited too long, aborting."
+       exit 2
+    fi
+    if cat ${SPLICE_LOG} | grep "[ERROR] Failed to execute goal" ; then
+      echo "ERROR: Spliceengine couldn't start."
+      exit 2
+    fi
+    sleep 2
+  done
+  echo "done!"
 
 }
 
@@ -234,7 +260,7 @@ function _start_mem
 
 function _wait_for_mem
 {
-  _wait_for_port "Waiting until Mem Platform is ready ..." 1527
+  _wait_for_port "Waiting until Mem Platform is ready ..." 1527 20
 }
 
 export -f _kill_em_all