Refactor cleanup task and add checks for gempak files

dev/ci/cases/pr/C48_S2SW.yaml

@@ -11,6 +11,9 @@ experiment:
   edate: 2021032312
   yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml
 
+skip_ci_on_hosts:
+  - wcoss2
+
 workflow:
   engine: rocoto
   rocoto:

dev/ci/cases/yamls/gfs_extended_ci.yaml

@@ -13,3 +13,4 @@ base:
   FHMAX_GFS: 384
   FHMAX_HF_GFS: 120
   DO_TEST_MODE: "NO"
+  DO_ARCHCOM: "YES"

dev/jobs/JGFS_ATMOS_POSTSND

@@ -12,6 +12,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "postsnd" -c "base postsnd"
 
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
+    COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
     COMOUT_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
     COMOUT_ATMOS_WMO:COM_ATMOS_WMO_TMPL \
     COMOUT_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL

dev/parm/config/gfs/config.cleanup

@@ -9,22 +9,40 @@ source "${EXPDIR}/config.resources" cleanup
 export CLEANUP_COM="YES"   # NO=retain ROTDIR.  YES default in cleanup.sh
 
 #--starting and ending hours of previous cycles to be removed from rotating directory
-export RMOLDSTD=144
-export RMOLDEND=24
+# RUN-based cleanup variables
+# Keep all files newer than SELECTIVE_CLEANUP_MIN hours 
+# Selectively remove files between SELECTIVE_CLEANUP_MAX and SELECTIVE_CLEANUP_MIN hours old, based on exclude_string
+# Remove all GEMPAK files older than GEMPAK_CLEANUP hours if DO_GEMPAK is set to YES and only for RUN==gfs
+# Remove **all** files and directories older than max(SELECTIVE_CLEANUP_MAX, GEMPAK_CLEANUP, RTOFS_CLEANUP)
 
-if [[ "${DO_GEMPAK}" == "YES" && "${RUN}" == "gfs" ]]; then
-    export RMOLDSTD=346
-    export RMOLDEND=222
+# RTOFS-specific variables (runs every RUN)
+# Remove all RTOFS files older than RTOFS_CLEANUP hours
+
+export SELECTIVE_CLEANUP_MAX=144
+export SELECTIVE_CLEANUP_MIN=24
+export GEMPAK_CLEANUP=240
+export RTOFS_CLEANUP=24
+if [[ ${SELECTIVE_CLEANUP_MIN} -gt ${SELECTIVE_CLEANUP_MAX} ]]; then
+    echo "FATAL ERROR: Invalid selective cleanup times: "
+    echo "             SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN} > SELECTIVE_CLEANUP_MAX=${SELECTIVE_CLEANUP_MAX}"
+    exit 1
+fi
+
+# GEMPAK_CLEANUP must not be less than SELECTIVE_CLEANUP_MIN, which acts as the starting point for cleanup
+if [[ "${DO_GEMPAK}" == "YES" && ${GEMPAK_CLEANUP} -lt ${SELECTIVE_CLEANUP_MIN} ]]; then
+    echo "FATAL ERROR: Invalid GEMPAK cleanup time: "
+    echo "             GEMPAK_CLEANUP=${GEMPAK_CLEANUP} < SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN}"
+    exit 2
 fi
 
 # Specify the list of files to exclude from the first stage of cleanup
 # Because arrays cannot be exported, list is a single string of comma-
 # separated values. This string is split to form an array at runtime.
+export selective_exclude_string=""
 case ${RUN} in
-	gdas | gfs)	exclude_string="*prepbufr*, *cnvstat.tar*, *analysis.atm.a*.nc" ;;
-	enkf*)      exclude_string="*f006.ens*" ;;
-	*)			exclude_string="" ;;
+    gdas | gfs)	selective_exclude_string+="*prepbufr*, *cnvstat.tar*, *analysis.atm.a*.nc" ;;
+    enkf*)      selective_exclude_string+="*f006.ens*" ;;
+    *)          selective_exclude_string="" ;;
 esac
-export exclude_string
 
 echo "END: config.cleanup"

dev/parm/config/gfs/config.resources

@@ -1378,7 +1378,7 @@ case ${step} in
     elif [[ "${RUN}" = *gfs ]]; then
       ntasks=28
       tasks_per_node=28
-      memory="4GB"
+      memory="6GB"
     fi
     ;;
 

dev/scripts/exgfs_atmos_postsnd.sh

@@ -149,20 +149,21 @@ else
 fi
 ${runscript} "${fhr}" "${fhr_p}" "${FINT}" "${F00FLAG}" "${DATA}"
 
-##############################################################
-# Tar and gzip the individual bufr files and send them to /com
-##############################################################
-cd "${COMOUT_ATMOS_BUFR}" || exit 2
-# shellcheck disable=SC2312
-tar -cf - . | /usr/bin/gzip > "${RUN}.${cycle}.bufrsnd.tar.gz"
-cd "${DATA}" || exit 2
+############################################
+# Tar and gzip the bufr files created so far
+############################################
+find "${COMIN_ATMOS_BUFR}" -maxdepth 1 -type f -name "bufr.*" -printf '%f\n' > "${DATA}/all_bufr_files.txt"
+
+tar -czf "${RUN}.${cycle}.soundings.tar.gz" -C "${COMIN_ATMOS_BUFR}" -T "${DATA}/all_bufr_files.txt"
+
+cpfs "${RUN}.${cycle}.soundings.tar.gz" "${COMOUT_ATMOS_BUFR}/"
 
 ########################################
 # Send the single tar file to OSO
 ########################################
 if [[ "${SENDDBN}" == "YES" ]]; then
     "${DBNROOT}/bin/dbn_alert" MODEL GFS_BUFRSND_TAR "${job}" \
-        "${COMOUT_ATMOS_BUFR}/${RUN}.${cycle}.bufrsnd.tar.gz"
+        "${COMOUT_ATMOS_BUFR}/${RUN}.${cycle}.soundings.tar.gz"
 fi
 
 ########################################

dev/scripts/exglobal_cleanup.sh

@@ -6,7 +6,9 @@ echo "Begin Cleanup ${DATAROOT}!"
 # Remove DATAoutput from the forecast model run
 # TODO: Handle this better
 DATAfcst="${DATAROOT}/${RUN}fcst.${PDY:-}${cyc}"
-if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
+if [[ -d "${DATAfcst}" ]]; then
+    rm -rf "${DATAfcst}"
+fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 ###############################################################
@@ -15,15 +17,48 @@ if [[ "${CLEANUP_COM:-YES}" == NO ]]; then
     exit 0
 fi
 
+SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN:-24}
+SELECTIVE_CLEANUP_MAX=${SELECTIVE_CLEANUP_MAX:-120}
+RTOFS_CLEANUP=${RTOFS_CLEANUP:-24}
+GEMPAK_CLEANUP=${GEMPAK_CLEANUP:-240}
 ###############################################################
 # Clean up previous cycles; various depths
 
 # Step back every assim_freq hours and remove old rotating directories
 # for successful cycles (defaults from 24h to 120h).
 # Retain files needed by Fit2Obs
-last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours")
-first_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
-last_rtofs=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDRTOFS:-48} hours")
+first_selective_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${SELECTIVE_CLEANUP_MAX} hours")
+last_selective_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${SELECTIVE_CLEANUP_MIN} hours")
+gempak_cutoff_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${GEMPAK_CLEANUP} hours")
+# Selective exclude list
+selective_exclude_string="${selective_exclude_string:-}"
+# Gempak exclude list
+gempak_exclude_string+=", gfs_1p00_*"
+
+# RTOFS is cleaned separately (for all RUNs)
+rtofs_cutoff_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RTOFS_CLEANUP} hours")
+
+# Find the first and last date among all cleanup targets
+max_cleanup_max="${SELECTIVE_CLEANUP_MAX:-120}"
+max_list="${SELECTIVE_CLEANUP_MAX}"
+if [[ "${RUN}" == "gfs" && "${DO_GEMPAK}" == "YES" ]]; then
+    max_list+=" ${GEMPAK_CLEANUP}"
+fi
+
+for cleanup_max in ${max_list}; do
+    if [[ ${cleanup_max} -gt ${max_cleanup_max} ]]; then
+        max_cleanup_max=${cleanup_max}
+    fi
+done
+
+# Start 4 cycles before the earliest exclusion target so we actually remove older files
+max_cleanup_max=$((max_cleanup_max + 4 * assim_freq))
+first_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${max_cleanup_max} hours")
+
+# Cleanup starts on SELECTIVE_CLEANUP_MIN
+last_date=${last_selective_date}
+
+# Declare remove_files(), which will be called in a loop to selectively remove files
 function remove_files() {
     local directory=$1
     shift
@@ -33,51 +68,62 @@ function remove_files() {
     fi
     local find_exclude_string=""
     for exclude in "$@"; do
-        find_exclude_string+="${find_exclude_string} -name ${exclude} -or"
+        if [[ -n "${exclude}" ]]; then
+            find_exclude_string+=" -name ${exclude} -or"
+        fi
     done
     # Chop off any trailing or
     find_exclude_string="${find_exclude_string[*]/%-or/}"
-    # Remove all regular files that do not match
-    # shellcheck disable=SC2086
-    if [[ -n "${find_exclude_string}" ]]; then
-        # String is non-empty → use exclusion
-        find "${directory}" -type f -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
-    else
-        # String is empty → no exclusion
-        find "${directory}" -type f -ignore_readdir_race -delete
-    fi
-
-    # Remove all symlinks that do not match
+    # Remove all regular files and symlinks that do not match
     # shellcheck disable=SC2086
     if [[ -n "${find_exclude_string}" ]]; then
         # String is non-empty → use exclusion
-        find "${directory}" -type l -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
+        find "${directory}" \( -type f -o -type l \) -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
     else
         # String is empty → no exclusion
-        find "${directory}" -type l -ignore_readdir_race -delete
+        find "${directory}" \( -type f -o -type l \) -ignore_readdir_race -delete
     fi
 
     # Remove any empty directories
     find "${directory}" -type d -empty -delete
 }
 
+# Now start removing old COM files/directories
 for ((current_date = first_date; current_date <= last_date;  \
 current_date = $(date --utc +%Y%m%d%H -d "${current_date:0:8} ${current_date:8:2} +${assim_freq} hours"))); do
     current_PDY="${current_date:0:8}"
     current_cyc="${current_date:8:2}"
-    rtofs_dir="${ROTDIR}/rtofs.${current_PDY}"
     rocotolog="${EXPDIR}/logs/${current_date}.log"
+
+    # Build the exclude list based on the 'current_date'
+    exclude_string=""
+    if [[ ${current_date} -ge ${first_selective_date} && ${current_date} -le ${last_selective_date} ]]; then
+        exclude_string+="${selective_exclude_string}"
+    fi
+
+    # Extend the exclude list for gempak files if needed
+    if [[ "${RUN}" == "gfs" && ${current_date} -ge ${gempak_cutoff_date} && "${DO_GEMPAK}" == "YES" ]]; then
+        # Provide the gempak exclude pattern(s)
+        exclude_string+="${gempak_exclude_string}"
+    fi
+
+    # Check if the cycle completed successfully by looking at the rocoto log
     if [[ -f "${rocotolog}" ]]; then
         # TODO: This needs to be revamped to not look at the rocoto log.
-        # shellcheck disable=SC2312
-        if [[ $(tail -n 1 "${rocotolog}") =~ "This cycle is complete: Success" ]]; then
+        tail_log=$(tail -n 1 "${rocotolog}") || true
+        # Test if the last line of rocotolog indicates success
+        if [[ ${tail_log} =~ "This cycle is complete: Success" ]]; then
             YMD="${current_PDY}" HH="${current_cyc}" declare_from_tmpl \
                 COMOUT_TOP:COM_TOP_TMPL
             if [[ -d "${COMOUT_TOP}" ]]; then
-                IFS=", " read -r -a exclude_list <<< "${exclude_string:-}"
+                IFS=", " read -r -a exclude_list <<< "${exclude_string}"
                 remove_files "${COMOUT_TOP}" "${exclude_list[@]:-}"
             fi
-            if [[ -d "${rtofs_dir}" ]] && ((current_date < last_rtofs)); then rm -rf "${rtofs_dir}"; fi
+            # Remove all rtofs directories in each RUN older than last_rtofs_date
+            rtofs_dir="${ROTDIR}/rtofs.${current_PDY}"
+            if [[ -d "${rtofs_dir}" && ${current_date} -lt ${rtofs_cutoff_date} ]]; then
+                rm -rf "${rtofs_dir}"
+            fi
         fi
     fi
 done
@@ -107,14 +153,17 @@ if [[ "${RUN}" == "gfs" ]]; then
     done
 fi
 
-# Remove $RUN.$rPDY for the older of GDATE or RDATE
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
+# Remove $RUN.$rPDY for the older of GDATE or RDATE if it is empty
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${max_cleanup_max} hours")
 RDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${FHMAX_GFS} hours")
 if ((GDATE < RDATE)); then
     RDATE=${GDATE}
 fi
+
 deletion_target="${ROTDIR}/${RUN}.${RDATE:0:8}"
-if [[ -d ${deletion_target} ]]; then rm -rf "${deletion_target}"; fi
+if [[ -d "${deletion_target}" ]]; then
+    find "${deletion_target}" -maxdepth 0 -type d -empty -delete
+fi
 
 # sync and wait to avoid filesystem synchronization issues
 sync && sleep 1

dev/workflow/rocoto/gefs_tasks.py

@@ -212,7 +212,7 @@ def _atmosoceaniceprod(self, component: str):
         fhout_ice_gfs = self._configs['base']['FHOUT_ICE_GFS']
         products_dict = {'atmos': {'config': 'atmos_products',
                                    'history_path_tmpl': 'COM_ATMOS_MASTER_TMPL',
-                                   'history_file_tmpl': f'{self.run}[email protected].grb2f#fhr3_last#'},
+                                   'history_file_tmpl': f'{self.run}[email protected].f#fhr3_last#.grib2'},
                          'ocean': {'config': 'oceanice_products',
                                    'history_path_tmpl': 'COM_OCEAN_HISTORY_TMPL',
                                    'history_file_tmpl': f'{self.run}.ocean.t@Hz.{fhout_ocn_gfs}hr_avg.f#fhr3_next#.nc'},

dev/workflow/rocoto/sfs_tasks.py

@@ -190,7 +190,7 @@ def _atmosoceaniceprod(self, component: str):
         fhout_ice_gfs = self._configs['base']['FHOUT_ICE_GFS']
         products_dict = {'atmos': {'config': 'atmos_products',
                                    'history_path_tmpl': 'COM_ATMOS_MASTER_TMPL',
-                                   'history_file_tmpl': f'{self.run}[email protected].grb2f#fhr3_last#'},
+                                   'history_file_tmpl': f'{self.run}[email protected].f#fhr3_last#.grib2'},
                          'ocean': {'config': 'oceanice_products',
                                    'history_path_tmpl': 'COM_OCEAN_HISTORY_TMPL',
                                    'history_file_tmpl': f'{self.run}.ocean.t@Hz.{fhout_ocn_gfs}hr_avg.f#fhr3_next#.nc'},

modulefiles/gw_setup.hera.lua

@@ -16,6 +16,5 @@ load(pathJoin("python", python_ver))
 load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
-load("gh")
 
 whatis("Description: GFS run setup environment")

modulefiles/gw_setup.hercules.lua

@@ -17,6 +17,5 @@ load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
 try_load("globus-cli")
-load("gh")
 
 whatis("Description: GFS run setup environment")

modulefiles/gw_setup.orion.lua

@@ -19,6 +19,5 @@ load("py-pyyaml")
 load("py-numpy")
 load(pathJoin("cmake", cmake_ver))
 try_load("globus-cli")
-load("gh")
 
 whatis("Description: GFS run setup environment")

modulefiles/gw_setup.ursa.lua

@@ -16,6 +16,5 @@ load(pathJoin("cmake", cmake_ver))
 load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
-load("gh")
 
 whatis("Description: GFS run setup environment")

parm/archive/gfs_downstream.yaml.j2

@@ -3,7 +3,7 @@ gfs_downstream:
     name: "GFS_DOWNSTREAM"
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar"
     required:
-        {% if DO_GEMPAK %}
+        {% if DO_GEMPAK and DO_BUFRSND %}
         - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc.bufr"
         - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.soundings.bufr"
         {% endif %}

ush/gfs_bfr2gpk.sh

@@ -30,10 +30,13 @@ date
 cat "${COMOUT_ATMOS_BUFR}/bufr."*".${PDY}${cyc}" > bufr.combined
 date
 
+snd=${outfilbase}.snd
+sfc=${outfilbase}.sfc
+
 namsnd << EOF > /dev/null
 SNBUFR   = bufr.combined
-SNOUTF   = ${outfilbase}.snd
-SFOUTF   = ${outfilbase}.sfc
+SNOUTF   = ${snd}
+SFOUTF   = ${sfc}
 SNPRMF   = sngfs.prm
 SFPRMF   = sfgfs.prm
 TIMSTN   = 170/2150
@@ -46,13 +49,13 @@ date
 
 /bin/rm ./*.nts
 
-snd=${outfilbase}.snd
-sfc=${outfilbase}.sfc
-cpfs "${snd}" "${COMOUT_ATMOS_GEMPAK}/${snd}"
-cpfs "${sfc}" "${COMOUT_ATMOS_GEMPAK}/${sfc}"
+snd_out=${outfilbase}.soundings.bufr
+sfc_out=${outfilbase}.sfc.bufr
+cpfs "${snd}" "${COMOUT_ATMOS_GEMPAK}/${snd_out}"
+cpfs "${sfc}" "${COMOUT_ATMOS_GEMPAK}/${sfc_out}"
 
 if [[ ${SENDDBN} == "YES" ]]; then
-   "${DBNROOT}/bin/dbn_alert" MODEL GFS_PTYP_SFC "${job}" "${COMOUT_ATMOS_GEMPAK}/${sfc}"
-   "${DBNROOT}/bin/dbn_alert" MODEL GFS_PTYP_SND "${job}" "${COMOUT_ATMOS_GEMPAK}/${snd}"
+   "${DBNROOT}/bin/dbn_alert" MODEL GFS_PTYP_SFC "${job}" "${COMOUT_ATMOS_GEMPAK}/${sfc_out}"
+   "${DBNROOT}/bin/dbn_alert" MODEL GFS_PTYP_SND "${job}" "${COMOUT_ATMOS_GEMPAK}/${snd_out}"
 fi
 echo "done" > "${DATA}/gembufr.done"