Refactor cleanup task and add checks for gempak files

dev/parm/config/gfs/config.cleanup

@@ -9,22 +9,30 @@ source "${EXPDIR}/config.resources" cleanup
 export CLEANUP_COM="YES"   # NO=retain ROTDIR.  YES default in cleanup.sh
 
 #--starting and ending hours of previous cycles to be removed from rotating directory
-export RMOLDSTD=144
-export RMOLDEND=24
+# Selectively remove files between SELECTIVE_RM_MIN and SELECTIVE_RM_MIN hours old, based on exclude_string
+# Remove all RTOFS files older than RTOFS_CLEANUP_MAX hours
+# Remove all files, except GEMPAK files, older than SELECTIVE_RM_MAX hours, based on gempak_exclude_string
+# Remove all files older than the max of all SELECTIVE_RM* variables
+# Retain all files newer than SELECTIVE_RM_MIN hours
 
-if [[ "${DO_GEMPAK}" == "YES" && "${RUN}" == "gfs" ]]; then
-    export RMOLDSTD=346
-    export RMOLDEND=222
+export SELECTIVE_CLEANUP_MAX=144
+export SELECTIVE_CLEANUP_MIN=24
+export GEMPAK_CLEANUP_MAX=240
+export RTOFS_CLEANUP_MAX=48
+if [[ ${SELECTIVE_CLEANUP_MIN} -gt ${SELECTIVE_CLEANUP_MAX} ]]; then
+    echo "FATAL ERROR: Invalid selective cleanup times: "
+    echo "             SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN} > SELECTIVE_CLEANUP_MAX=${SELECTIVE_CLEANUP_MAX}"
+    exit 1
 fi
 
 # Specify the list of files to exclude from the first stage of cleanup
 # Because arrays cannot be exported, list is a single string of comma-
 # separated values. This string is split to form an array at runtime.
+export exclude_string=""
 case ${RUN} in
-	gdas | gfs)	exclude_string="*prepbufr*, *cnvstat.tar*, *analysis.atm.a*.nc" ;;
-	enkf*)      exclude_string="*f006.ens*" ;;
+	gdas | gfs)	exclude_string+="*prepbufr*, *cnvstat.tar*, *analysis.atm.a*.nc" ;;
+	enkf*)      exclude_string+="*f006.ens*" ;;
 	*)			exclude_string="" ;;
 esac
-export exclude_string
 
 echo "END: config.cleanup"

scripts/exglobal_cleanup.sh

@@ -6,78 +6,107 @@
 # Remove DATAoutput from the forecast model run
 # TODO: Handle this better
 DATAfcst="${DATAROOT}/${RUN}fcst.${PDY:-}${cyc}"
-if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
+if [[ -d "${DATAfcst}" ]];
+    then rm -rf "${DATAfcst}";
+fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 ###############################################################
 
 if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
     exit 0
 fi
 
+SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN:-24}
+SELECTIVE_CLEANUP_MAX=${SELECTIVE_CLEANUP_MAX:-120}
+RTOFS_CLEANUP_MAX=${RTOFS_CLEANUP_MAX:-48}
+GEMPAK_CLEANUP_MAX=${GEMPAK_CLEANUP_MAX:-240}
 ###############################################################
 # Clean up previous cycles; various depths
 
 # Step back every assim_freq hours and remove old rotating directories
 # for successful cycles (defaults from 24h to 120h).
 # Retain files needed by Fit2Obs
-last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours")
-first_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
-last_rtofs=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDRTOFS:-48} hours")
+last_selective_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${SELECTIVE_CLEANUP_MIN} hours")
+first_selective_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${SELECTIVE_CLEANUP_MAX} hours")
+last_rtofs_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RTOFS_CLEANUP_MAX} hours")
+last_gempak_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${GEMPAK_CLEANUP_MAX} hours")
+exclude_string="${exclude_string:-}"
+
+# Find the last date among all cleanup targets
+max_cleanup_max="${SELECTIVE_CLEANUP_MAX:-120}"
+for cleanup_max in "${RTOFS_CLEANUP_MAX}" "${GEMPAK_CLEANUP_MAX}"; do
+    if [[ ${cleanup_max} -gt ${max_cleanup_max} ]]; then
+        max_cleanup_max=${cleanup_max}
+    fi
+done
+
+last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${max_cleanup_max} hours")
+
 function remove_files() {
     local directory=$1
     shift
     if [[ ! -d ${directory} ]]; then
         echo "No directory ${directory} to remove files from, skiping"
         return
     fi
-    local find_exclude_string=""
-    for exclude in "$@"; do
-        find_exclude_string+="${find_exclude_string} -name ${exclude} -or"
+    # Find all files and links in the directory and store as an arry
+    # Run find only once for efficiency
+    flist=($(find "${directory}" -type f -or -type l))
+
+    # Now remove those files that match the exclude patterns
+    for exclude_pattern in "$@"; do
+        # Use a temporary array to hold files that do not match the exclude pattern
+        temp_flist=()
+        for file in "${flist[@]}"; do
+            case "$(basename "${file}")" in
+                ${exclude_pattern})
+                    # Match found, skip this file
+                    ;;
+                *)
+                    # No match, keep this file
+                    temp_flist+=("${file}")
+                    ;;
+            esac
+        done
+        flist=("${temp_flist[@]}")
     done
-    # Chop off any trailing or
-    find_exclude_string="${find_exclude_string[*]/%-or}"
-    # Remove all regular files that do not match
-    # shellcheck disable=SC2086
-    if [[ -n "${find_exclude_string}" ]]; then
-    # String is non-empty → use exclusion
-       find "${directory}" -type f -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
-    else
-    # String is empty → no exclusion
-       find "${directory}" -type f -ignore_readdir_race -delete
-    fi
 
-    # Remove all symlinks that do not match
-    # shellcheck disable=SC2086
-    if [[ -n "${find_exclude_string}" ]]; then
-    # String is non-empty → use exclusion
-       find "${directory}" -type l -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
-    else
-    # String is empty → no exclusion
-       find "${directory}" -type l -ignore_readdir_race -delete
-    fi
+    # Delete all files in flist.
+    for file in "${flist[@]}"; do
+        rm -f "${file}"
+    done
 
     # Remove any empty directories
     find "${directory}" -type d -empty -delete
 }
 
-for (( current_date=first_date; current_date <= last_date; \
+# Now start removing old COM files/directories
+for (( current_date=first_selective_date; current_date <= last_date; \
   current_date=$(date --utc +%Y%m%d%H -d "${current_date:0:8} ${current_date:8:2} +${assim_freq} hours") )); do
     current_PDY="${current_date:0:8}"
     current_cyc="${current_date:8:2}"
-    rtofs_dir="${ROTDIR}/rtofs.${current_PDY}"
     rocotolog="${EXPDIR}/logs/${current_date}.log"
+
+    # Extend the exclude list for gempak files if needed
+    if [[ "${RUN}" == "gfs" && ${current_date} -lt ${last_gempak_date} && "${DO_GEMPAK}" == "YES" ]]; then
+        # Provide the gempak exclude pattern(s)
+        exclude_string+=", *gfs_1p00_*"
+    fi
+
+    # Check if the cycle completed successfully by looking at the rocoto log
     if [[ -f "${rocotolog}" ]]; then
         # TODO: This needs to be revamped to not look at the rocoto log.
-        # shellcheck disable=SC2312
         if [[ $(tail -n 1 "${rocotolog}") =~ "This cycle is complete: Success" ]]; then
             YMD="${current_PDY}" HH="${current_cyc}" declare_from_tmpl \
                 COMOUT_TOP:COM_TOP_TMPL
             if [[ -d "${COMOUT_TOP}" ]]; then
-                IFS=", " read -r -a exclude_list <<< "${exclude_string:-}"
+                IFS=", " read -r -a exclude_list <<< "${exclude_string}"
                 remove_files "${COMOUT_TOP}" "${exclude_list[@]:-}"
             fi
-            if [[ -d "${rtofs_dir}" ]] && (( current_date < last_rtofs )); then rm -rf "${rtofs_dir}" ; fi
+            # Remove all rtofs directories in each RUN older than last_rtofs_date
+            rtofs_dir="${ROTDIR}/rtofs.${current_PDY}"
+            if [[ -d "${rtofs_dir}" ]] && (( current_date < last_rtofs_date )); then rm -rf "${rtofs_dir}" ; fi
         fi
     fi
 done
@@ -108,13 +137,16 @@
 fi
 
 # Remove $RUN.$rPDY for the older of GDATE or RDATE
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${max_cleanup_max} hours")
 RDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${FHMAX_GFS} hours")
 if (( GDATE < RDATE )); then
     RDATE=${GDATE}
 fi
+
 deletion_target="${ROTDIR}/${RUN}.${RDATE:0:8}"
-if [[ -d ${deletion_target} ]]; then rm -rf "${deletion_target}"; fi
+if [[ -d "${deletion_target}" ]]; then
+    rm -rf "${deletion_target}"
+fi
 
 # sync and wait to avoid filesystem synchronization issues
 sync && sleep 1

ush/preamble.sh

@@ -33,8 +33,8 @@ declare -x PS4='+ $(basename ${BASH_SOURCE[0]:-${FUNCNAME[0]:-"Unknown"}})[${LIN
 
 set_strict() {
     if [[ ${STRICT:-"YES"} == "YES" ]]; then
-        # Exit on error and undefined variable
-        set -eu
+        # Exit on error, undefined variable, or error in a pipeline (e.g. if and command in "cmd | cmd2" fails)
+        set -euo pipefail
     fi
 }