diff --git a/CHANGES.md b/CHANGES.md index 7015b679a..313f2b140 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,14 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +- Introduce a scheduler job to delete old files that may accumulate over time + + Creates the `optional-component/clean_old_files` job that deletes old THREDDS log files and WPS output files. + Allows individual cleanup jobs to be enabled for each of `raven`, `finch`, `hummingbird`, and `thredds` components. + Allows the user to configure how old a file must be before it is deleted (age in days) and how to calculate the age + of the file (time since last modified, time since last accessed, time since created). + + (see `env.local.example` or the `scheduler` documentation for details). [2.21.0](https://github.com/bird-house/birdhouse-deploy/tree/2.21.0) (2026-01-27) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/README.rst b/birdhouse/components/README.rst index a09636408..c52b50f23 100644 --- a/birdhouse/components/README.rst +++ b/birdhouse/components/README.rst @@ -59,6 +59,15 @@ component directory to the ``BIRDHOUSE_EXTRA_CONF_DIRS`` variable in your local * component location: ``optional-components/scheduler-job-deploy_raven_testdata`` +* Automatically remove old files + + * Removes files generated by other components that may accumulate over time and are not manage automatically by those components. + + * Currently supports removing WPS output files from the ``finch``, ``raven``, and ``hummingbird`` components as well as log files + from the ``thredds`` component. + + * component location: ``optional-components/scheduler-job-clean_old_files`` + * Automatically backup files * automatically backs up application data, user data, representative data, and logs to a restic repository (default) or a docker volume diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 47d922c15..b114a1e8e 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -204,6 +204,43 @@ export GEOSERVER_ADMIN_PASSWORD="${__DEFAULT__GEOSERVER_ADMIN_PASSWORD}" # (note: if using 'BIRDHOUSE_DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env') #export BIRDHOUSE_LOGROTATE_DATA_DIR='${BIRDHOUSE_DATA_PERSIST_ROOT}/logrotate' +# These variables configure the scheduler-job-clean_old_files component +# +# For all options below: +# - variables that end with SCHEDULER_JOB_CLEAN_OLD_FILES_ENABLED will enable the a clean_old_files job for the relevant component if set to 'true' +# - variables that end with DELETE_FILES_OLDER_THAN_DAYS set a number of days. Files older than this number of days will be deleted every time +# the scheduler-job-clean_old_files scheduler job runs. +# - variables that end with DELETE_FILES_TIME_MODE is used by the find command to calculate the age of a file: +# - atime: delete files that haven't been accessed in X days +# - mtime: delete files that haven't been modified in X days +# - ctime: delete files that were created more than X days ago +# - variables that end with SCHEDULER_JOB_CLEAN_OLD_FILES_FREQUENCY set the frequency at which the relevant clean old files job should be run. This +# value is a string that conforms to the cron schedule format. +# +# Delete old WPS output files generated by the finch WPS component +#export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +#export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_TIME_MODE=atime +#export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_FREQUENCY="5 4 * * 0" # weekly on Sunday at 4:05 +#export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_ENABLED=false +# +# Delete old WPS output files generated by the hummingbird WPS component +#export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +#export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_TIME_MODE=atime +#export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_FREQUENCY="10 4 * * 0" # weekly on Sunday at 4:10 +#export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_ENABLED=false +# +# Delete old WPS output files generated by the raven WPS component +#export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +#export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_TIME_MODE=atime +#export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_FREQUENCY="10 4 * * 0" # weekly on Sunday at 4:10 +#export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_ENABLED=false +# +# Delete old log files generated by the thredds component +#export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +#export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_TIME_MODE=mtime +#export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_FREQUENCY="20 4 * * 0" # weekly on Sunday at 4:20 +#export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_ENABLED=false + # These variables configure the scheduler-job-backup component # Schedule for when to run the job to automatically back up data diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/.gitignore b/birdhouse/optional-components/scheduler-job-clean_old_files/.gitignore new file mode 100644 index 000000000..1d3ed4c17 --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/.gitignore @@ -0,0 +1 @@ +config.yml diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/blank.config.yml b/birdhouse/optional-components/scheduler-job-clean_old_files/blank.config.yml new file mode 100644 index 000000000..03b83d2d7 --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/blank.config.yml @@ -0,0 +1 @@ +# this file intentionally contains no content and is mounted to the scheduler directory if a clean_old_files job is not enabled. diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/clean-old-files.sh b/birdhouse/optional-components/scheduler-job-clean_old_files/clean-old-files.sh new file mode 100755 index 000000000..37d30032d --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/clean-old-files.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +################################################################ +# Example call to delete all files in /tmp last modified longer +# than 20 days ago +# +# $ sh clean-old-files.sh 20 mtime /tmp +################################################################## + +AGE="$1" +MODE="$2" +LOCATION="$3" + +ACCEPTABLE_MODES='|mtime|ctime|atime|' + +if ! echo "$AGE" | grep -q '^[0-9][0-9]*$'; then + >&2 echo "AGE argument set to '${AGE}'. It must be an unsigned integer" + exit 1 +fi + +if [ "${ACCEPTABLE_MODES#*"|${MODE}|"}" = "${ACCEPTABLE_MODES}" ]; then + >&2 echo "MODE argument set to '${MODE}'. It must be one of 'mtime', 'ctime', or 'atime'" + exit 1 +fi + +if [ -z "${LOCATION}" ]; then + >&2 echo "LOCATION argument is blank or unset. It must refer to a path on disk." + exit 1 +fi + +echo "Removing files in ${LOCATION} that have a ${MODE} value greater than ${AGE} days" +find "${LOCATION}" -type f "-${MODE}" +"${AGE}" -print -delete diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/finch/config.yml.template b/birdhouse/optional-components/scheduler-job-clean_old_files/config/finch/config.yml.template new file mode 100644 index 000000000..5c71c2308 --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/finch/config.yml.template @@ -0,0 +1,9 @@ +- name: clean_old_files_finch + comment: clean old WPS output files generated by Finch + schedule: '${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_FREQUENCY}' + command: 'sh /clean-old-files.sh "${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_OLDER_THAN_DAYS}" "${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_TIME_MODE}" /wps_outputs/finch' + dockerargs: >- + --rm --name scheduler-job-clean_old_files_finch + --volume ${COMPOSE_DIR}/optional-components/scheduler-job-clean_old_files/clean-old-files.sh:/clean-old-files.sh:ro + --volume "${COMPOSE_PROJECT_NAME}_wps_outputs:/wps_outputs:rw" + image: '${SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE}' diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/finch/docker-compose-extra.yml b/birdhouse/optional-components/scheduler-job-clean_old_files/config/finch/docker-compose-extra.yml new file mode 100644 index 000000000..fa8610e1c --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/finch/docker-compose-extra.yml @@ -0,0 +1,4 @@ +services: + scheduler: + volumes: + - ./optional-components/scheduler-job-clean_old_files/${__SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_CONFIG_LOC:-blank.config.yml}:/scheduler-job-configs/clean_old_files_finch.yml:ro} diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/hummingbird/config.yml.template b/birdhouse/optional-components/scheduler-job-clean_old_files/config/hummingbird/config.yml.template new file mode 100644 index 000000000..bd971b8df --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/hummingbird/config.yml.template @@ -0,0 +1,9 @@ +- name: clean_old_files_hummingbird + comment: clean old WPS output files generated by Hummingbird + schedule: '${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_FREQUENCY}' + command: 'sh /clean-old-files.sh "${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_OLDER_THAN_DAYS}" "${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_TIME_MODE}" /wps_outputs/hummingbird' + dockerargs: >- + --rm --name scheduler-job-clean_old_files_hummingbird + --volume ${COMPOSE_DIR}/optional-components/scheduler-job-clean_old_files/clean-old-files.sh:/clean-old-files.sh:ro + --volume "${COMPOSE_PROJECT_NAME}_wps_outputs:/wps_outputs:rw" + image: '${SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE}' diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/hummingbird/docker-compose-extra.yml b/birdhouse/optional-components/scheduler-job-clean_old_files/config/hummingbird/docker-compose-extra.yml new file mode 100644 index 000000000..f9b06024c --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/hummingbird/docker-compose-extra.yml @@ -0,0 +1,4 @@ +services: + scheduler: + volumes: + - ./optional-components/scheduler-job-clean_old_files/${__SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_CONFIG_LOC:-blank.config.yml}:/scheduler-job-configs/clean_old_files_hummingbird.yml:ro} diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/raven/config.yml.template b/birdhouse/optional-components/scheduler-job-clean_old_files/config/raven/config.yml.template new file mode 100644 index 000000000..22b1c85f8 --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/raven/config.yml.template @@ -0,0 +1,9 @@ +- name: clean_old_files_raven + comment: clean old WPS output files generated by Raven + schedule: '${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_FREQUENCY}' + command: 'sh /clean-old-files.sh "${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_OLDER_THAN_DAYS}" "${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_TIME_MODE}" /wps_outputs/raven' + dockerargs: >- + --rm --name scheduler-job-clean_old_files_raven + --volume ${COMPOSE_DIR}/optional-components/scheduler-job-clean_old_files/clean-old-files.sh:/clean-old-files.sh:ro + --volume "${COMPOSE_PROJECT_NAME}_wps_outputs:/wps_outputs:rw" + image: '${SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE}' diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/raven/docker-compose-extra.yml b/birdhouse/optional-components/scheduler-job-clean_old_files/config/raven/docker-compose-extra.yml new file mode 100644 index 000000000..8e85addbd --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/raven/docker-compose-extra.yml @@ -0,0 +1,4 @@ +services: + scheduler: + volumes: + - ./optional-components/scheduler-job-clean_old_files/${__SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_CONFIG_LOC:-blank.config.yml}:/scheduler-job-configs/clean_old_files_raven.yml:ro} diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/thredds/config.yml.template b/birdhouse/optional-components/scheduler-job-clean_old_files/config/thredds/config.yml.template new file mode 100644 index 000000000..f7adb08c3 --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/thredds/config.yml.template @@ -0,0 +1,9 @@ +- name: clean_old_files_thredds + comment: clean old log files generated by Thredds + schedule: '${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_FREQUENCY}' + command: 'sh /clean-old-files.sh "${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_OLDER_THAN_DAYS}" "${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_TIME_MODE}" /thredds' + dockerargs: >- + --rm --name scheduler-job-clean_old_files_thredds + --volume ${COMPOSE_DIR}/optional-components/scheduler-job-clean_old_files/clean-old-files.sh:/clean-old-files.sh:ro + --volume "thredds_persistence:/thredds:rw" + image: '${SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE}' diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/config/thredds/docker-compose-extra.yml b/birdhouse/optional-components/scheduler-job-clean_old_files/config/thredds/docker-compose-extra.yml new file mode 100644 index 000000000..61440f7e9 --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/config/thredds/docker-compose-extra.yml @@ -0,0 +1,4 @@ +services: + scheduler: + volumes: + - ./optional-components/scheduler-job-clean_old_files/${__SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_CONFIG_LOC:-blank.config.yml}:/scheduler-job-configs/clean_old_files_thredds.yml:ro} diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/default.env b/birdhouse/optional-components/scheduler-job-clean_old_files/default.env new file mode 100644 index 000000000..9e0bde48b --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/default.env @@ -0,0 +1,57 @@ +export SCHEDULER_JOB_CLEAN_OLD_FILES_DOCKER=alpine # alpine contains find with -ctime -mtime and -atime options (busybox based containers do not) +export SCHEDULER_JOB_CLEAN_OLD_FILES_VERSION=3.21 +export SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE='${SCHEDULER_JOB_CLEAN_OLD_FILES_DOCKER}:${SCHEDULER_JOB_CLEAN_OLD_FILES_VERSION}' + +export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_TIME_MODE=atime +export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_FREQUENCY="5 4 * * 0" # weekly on Sunday at 4:05 +export SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_ENABLED=false +export __SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_CONFIG_LOC='$( [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_ENABLED}" = "true" ] && echo "config/finch/config.yml" )' + +export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_TIME_MODE=atime +export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_FREQUENCY="10 4 * * 0" # weekly on Sunday at 4:10 +export SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_ENABLED=false +export __SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_CONFIG_LOC='$( [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_ENABLED}" = "true" ] && echo "config/hummingbird/config.yml" )' + +export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_TIME_MODE=atime +export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_FREQUENCY="15 4 * * 0" # weekly on Sunday at 4:15 +export SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_ENABLED=false +export __SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_CONFIG_LOC='$( [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_ENABLED}" = "true" ] && echo "config/raven/config.yml" )' + +export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_OLDER_THAN_DAYS= # unset by default if this job is enabled this must be set to an integer +export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_TIME_MODE=mtime +export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_FREQUENCY="20 4 * * 0" # weekly on Sunday at 4:20 +export SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_ENABLED=false +export __SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_CONFIG_LOC='$( [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_ENABLED}" = "true" ] && echo "config/thredds/config.yml" )' + +export DELAYED_EVAL=" + $DELAYED_EVAL + SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE + __SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_CONFIG_LOC + __SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_CONFIG_LOC + __SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_CONFIG_LOC + __SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_CONFIG_LOC +" + +OPTIONAL_VARS=" + $OPTIONAL_VARS + \$SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_OLDER_THAN_DAYS + \$SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_TIME_MODE + \$SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_FREQUENCY + \$SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_OLDER_THAN_DAYS + \$SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_TIME_MODE + \$SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_FREQUENCY + \$SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_OLDER_THAN_DAYS + \$SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_TIME_MODE + \$SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_FREQUENCY + \$SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_OLDER_THAN_DAYS + \$SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_TIME_MODE + \$SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_FREQUENCY +" + +VARS=" + $VARS + \$SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE +" diff --git a/birdhouse/optional-components/scheduler-job-clean_old_files/pre-docker-compose-up.include b/birdhouse/optional-components/scheduler-job-clean_old_files/pre-docker-compose-up.include new file mode 100644 index 000000000..5194d5cd3 --- /dev/null +++ b/birdhouse/optional-components/scheduler-job-clean_old_files/pre-docker-compose-up.include @@ -0,0 +1,25 @@ +if ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'scheduler[[:space:]]*$'; then + log WARN 'The scheduler-job-clean_old_files component is enabled but the scheduler component is not. This WILL cause problems. Please disable the scheduler-job-clean_old_files component.' +fi + +_acceptable_modes='|mtime|ctime|atime|' + +if [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_ENABLED}" = "true" ] && ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'finch[[:space:]]*$'; then + echo "$SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_OLDER_THAN_DAYS" | grep -q '^[0-9][0-9]*$' || log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_OLDER_THAN_DAYS variable must be an integer not '${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_OLDER_THAN_DAYS}'. Please set this variable to an integer or disable the finch file cleaning job. This job will not run properly!" + [ "${_acceptable_modes#*"${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_TIME_MODE}"}" = "${_acceptable_modes}" ] && log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_TIME_MODE variable must be one of 'mtime', 'atime', or 'ctime' not '${SCHEDULER_JOB_CLEAN_OLD_FILES_FINCH_TIME_MODE}'. Please set this variable to a valid option or disable the finch file cleaning job. This job will not run properly!" +fi + +if [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_ENABLED}" = "true" ] && ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'hummingbird[[:space:]]*$'; then + echo "$SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_OLDER_THAN_DAYS" | grep -q '^[0-9][0-9]*$' || log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_OLDER_THAN_DAYS variable must be an integer not '${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_OLDER_THAN_DAYS}'. Please set this variable to an integer or disable the hummingbird file cleaning job. This job will not run properly!" + [ "${_acceptable_modes#*"${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_TIME_MODE}"}" = "${_acceptable_modes}" ] && log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_TIME_MODE variable must be one of 'mtime', 'atime', or 'ctime' not '${SCHEDULER_JOB_CLEAN_OLD_FILES_HUMMINGBIRD_TIME_MODE}'. Please set this variable to a valid option or disable the hummingbird file cleaning job. This job will not run properly!" +fi + +if [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_ENABLED}" = "true" ] && ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'raven[[:space:]]*$'; then + echo "$SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_OLDER_THAN_DAYS" | grep -q '^[0-9][0-9]*$' || log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_OLDER_THAN_DAYS variable must be an integer not '${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_OLDER_THAN_DAYS}'. Please set this variable to an integer or disable the raven file cleaning job. This job will not run properly!" + [ "${_acceptable_modes#*"${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_TIME_MODE}"}" = "${_acceptable_modes}" ] && log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_TIME_MODE variable must be one of 'mtime', 'atime', or 'ctime' not '${SCHEDULER_JOB_CLEAN_OLD_FILES_RAVEN_TIME_MODE}'. Please set this variable to a valid option or disable the raven file cleaning job. This job will not run properly!" +fi + +if [ "${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_ENABLED}" = "true" ] && ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'thredds[[:space:]]*$'; then + echo "$SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_OLDER_THAN_DAYS" | grep -q '^[0-9][0-9]*$' || log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_OLDER_THAN_DAYS variable must be an integer not '${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_OLDER_THAN_DAYS}'. Please set this variable to an integer or disable the thredds file cleaning job. This job will not run properly!" + [ "${_acceptable_modes#*"${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_TIME_MODE}"}" = "${_acceptable_modes}" ] && log WARN "SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_TIME_MODE variable must be one of 'mtime', 'atime', or 'ctime' not '${SCHEDULER_JOB_CLEAN_OLD_FILES_THREDDS_TIME_MODE}'. Please set this variable to a valid option or disable the thredds file cleaning job. This job will not run properly!" +fi