|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +# Ensure that a command is provided as an argument |
| 4 | +if [ $# -eq 0 ]; then |
| 5 | + echo "No command provided. Usage: ./run_until_fail.sh <command>" |
| 6 | + exit 1 |
| 7 | +fi |
| 8 | + |
| 9 | +# Command to run (passed as script arguments) |
| 10 | +COMMAND="$@" |
| 11 | +MAX_RETRIES=10 # Set the maximum number of retries |
| 12 | +COUNTER=0 # Initialize the retry counter |
| 13 | +CHECK_INTERVAL=5 # Interval in seconds to check if the command is still running |
| 14 | +ALERT_INTERVAL=120 # Interval in seconds to run jcmd if the command is still running |
| 15 | + |
| 16 | +while [ $COUNTER -lt $MAX_RETRIES ]; do |
| 17 | + # Increment the retry counter |
| 18 | + COUNTER=$((COUNTER + 1)) |
| 19 | + echo "Attempt $COUNTER of $MAX_RETRIES..." |
| 20 | + |
| 21 | + # Start the command in the background |
| 22 | + bash -c "$COMMAND" & |
| 23 | + COMMAND_PID=$! |
| 24 | + |
| 25 | + # Track elapsed time |
| 26 | + ELAPSED_TIME=0 |
| 27 | + |
| 28 | + # Check periodically if the command has completed |
| 29 | + while ps -p $COMMAND_PID > /dev/null; do |
| 30 | + sleep $CHECK_INTERVAL |
| 31 | + ELAPSED_TIME=$((ELAPSED_TIME + CHECK_INTERVAL)) |
| 32 | + |
| 33 | + # Run jcmd every ALERT_INTERVAL (60 seconds) |
| 34 | + if [ $((ELAPSED_TIME % ALERT_INTERVAL)) -eq 0 ]; then |
| 35 | + echo "Warning: The command has been running for more than $ELAPSED_TIME seconds." |
| 36 | + echo "\n\n----------------------------------------------------------------------------------------\n\n" |
| 37 | + |
| 38 | + # Run the jcmd command to print thread information |
| 39 | + jcmd $(jps -v | grep SpockTestConfig.groovy | awk '{print $1}') Thread.print |
| 40 | + echo "\n\n----------------------------------------------------------------------------------------\n\n" |
| 41 | + fi |
| 42 | + done |
| 43 | + |
| 44 | + # Capture the exit status |
| 45 | + wait $COMMAND_PID |
| 46 | + STATUS=$? |
| 47 | + |
| 48 | + # Check if the command failed |
| 49 | + if [ $STATUS -ne 0 ]; then |
| 50 | + echo "The command failed with exit status $STATUS. Exiting loop." |
| 51 | + break |
| 52 | + fi |
| 53 | + |
| 54 | + # Check if the retry limit has been reached |
| 55 | + if [ $COUNTER -ge $MAX_RETRIES ]; then |
| 56 | + echo "Reached the maximum number of retries ($MAX_RETRIES). Exiting." |
| 57 | + break |
| 58 | + fi |
| 59 | +done |
0 commit comments