Introduce a scheduler job to delete old files that may accumulate over time

CHANGES.md

@@ -60,6 +60,13 @@
       - Answer: This is a hack that would work based on the specific way that the docker-crontab image sets schedules.
                 However, this is not obvious to the user and is unreliable since it is not documented.
 
+- Introduce a scheduler job to delete old files that may accumulate over time
+
+  Creates the `optional-component-clean_old_files` job that deletes old THREDDS log files and WPS output files.
+  To set the oldest file that will be kept for each of these options, set the `THREDDS_DELETE_FILES_OLDER_THAN_DAYS`
+  and/or the `WPS_OUTPUTS_DELETE_FILES_OLDER_THAN_DAYS` variables in the local environment files (see 
+  `env.local.example` or the `scheduler` documentation for details).
+
 [2.10.1](https://github.com/bird-house/birdhouse-deploy/tree/2.10.1) (2025-03-10)
 ------------------------------------------------------------------------------------------------------------------
 

birdhouse/components/README.rst

@@ -59,6 +59,21 @@ component directory to the ``BIRDHOUSE_EXTRA_CONF_DIRS`` variable in your local
 
   * component location: ``optional-components/scheduler-job-deploy_raven_testdata``
 
+* Automatically remove old files
+
+  * Removes THREDDS log files and WPS output files older than a specific number of days
+
+  * In order to remove THREDDS log files the ``thredds`` component needs to be enabled.
+
+    * Set the ``THREDDS_DELETE_FILES_OLDER_THAN_DAYS`` variable in the local environment file to an integer specifying
+      how old a THREDDS log file needs to be before it is deleted.
+
+  * In order to remove WPS output files, at least one of the WPS components needs to be enabled.
+
+    * Set the ``WPS_OUTPUTS_DELETE_FILES_OLDER_THAN_DAYS`` variable in the local environment file to an integer specifying
+      how old a WPS output file needs to be before it is deleted.
+
+
 For additional configuration options for all these jobs see the ``env.local.example`` file
 as well as the individual ``default.env`` files in each of the component directories.
 

birdhouse/components/thredds/default.env

@@ -38,6 +38,12 @@ export THREDDS_DATASET_DATASETSCAN_BODY='
       </filter>
 '
 
+export THREDDS_DELETE_FILES_OLDER_THAN_DAYS=
+export SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS="
+  ${SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS}
+  \$([ -n \"\${THREDDS_DELETE_FILES_OLDER_THAN_DAYS}\" ] && echo \"thredds_persistence:/thredds|/thredds/logs/threddsServlet.*.log|\${THREDDS_DELETE_FILES_OLDER_THAN_DAYS}\")
+"
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 VARS="
   $VARS

birdhouse/components/wps_outputs-volume/default.env

@@ -3,6 +3,12 @@ OPTIONAL_VARS="
   \$SECURE_DATA_PROXY_AUTH_INCLUDE
 "
 
+export WPS_OUTPUTS_DELETE_FILES_OLDER_THAN_DAYS=
+export SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS="
+  ${SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS}
+  \$([ -n \"\${WPS_OUTPUTS_DELETE_FILES_OLDER_THAN_DAYS}\" ] && echo \"\${COMPOSE_PROJECT_NAME:-birdhouse}_wps_outputs:/wps_outputs|/wps_outputs|\${WPS_OUTPUTS_DELETE_FILES_OLDER_THAN_DAYS}\")
+"
+
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
 # single quotes are important in below list to keep variable names intact until 'birdhouse-compose' parses them
 EXTRA_VARS='

birdhouse/env.local.example

@@ -204,6 +204,22 @@ export GEOSERVER_ADMIN_PASSWORD="${__DEFAULT__GEOSERVER_ADMIN_PASSWORD}"
 # (note: if using 'BIRDHOUSE_DATA_PERSIST_ROOT', it must be defined earlier, either in this file or from 'default.env')
 #export BIRDHOUSE_LOGROTATE_DATA_DIR='${BIRDHOUSE_DATA_PERSIST_ROOT}/logrotate'
 
+# These variables configure the scheduler-job-clean_old_files component
+#
+# Delete THREDDS log files older than X days (e.g. X=20):
+#export THREDDS_DELETE_FILES_OLDER_THAN_DAYS=20
+#
+# Delete WPS output files older than X days (e.g. X=90):
+#export WPS_OUTPUTS_DELETE_FILES_OLDER_THAN_DAYS=90
+# 
+# Set cron schedule for the clean old files job (how often the job runs).
+# By default it runs weekly on Sunday at 2:05 am:
+#export SCHEDULER_JOB_CLEAN_OLD_FILES_FREQUENCY="5 2 * * 0"
+
+#############################################################
+# Proxy variables
+#############################################################
+
 # Content of "location /" in file config/proxy/conf.d/all-services.include.template
 # Useful to have a custom homepage.
 # Default:

birdhouse/optional-components/scheduler-job-clean_old_files/.gitignore

@@ -0,0 +1 @@
+config.yml

birdhouse/optional-components/scheduler-job-clean_old_files/clean-old-files.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+################################################################
+# Deletes old files as determined by the CLEAN_OLD_FILES_OPTIONS
+# environment variable.
+#
+# This variable contains space delimited fields, each
+# representing a group of files to be deleted.
+# The format of these fields are as follows:
+#
+# <docker-volume-mounts>|<find-location>|<age-in-days>
+#
+# - docker-volume-mounts is not used by this script
+# - find-location is an argument passed to `find` which will
+#   recursively search for files to delete based on that argument
+# - age-in-days is an integer that represents a number of days,
+#   all files found by `find` that were modified more than this
+#   number of days ago will be deleted
+#
+# Example call to delete all files in /tmp older than 20 days and
+# all files in /var/log older than 90 days:
+#
+# $ export CLEAN_OLD_FILES_OPTIONS='xxx|/tmp|20 yyy|/var/log|90' 
+# $ sh clean-old-files.sh
+##################################################################
+
+
+for opt in ${CLEAN_OLD_FILES_OPTIONS}; do
+    loc="$(echo $opt | cut -d\| -f 2)"
+    age="$(echo $opt | cut -d\| -f 3)"
+    echo "Removing files in ${loc} that have not been modified in ${age} days"
+    [ -n "$loc" ] && [ -n "$age" ] && find ${loc} -type f -mtime +"${age}" -print -delete
+done

birdhouse/optional-components/scheduler-job-clean_old_files/config.yml.template

@@ -0,0 +1,9 @@
+- name: clean_old_files
+  comment: clean old files generated by the stack
+  schedule: '${SCHEDULER_JOB_CLEAN_OLD_FILES_FREQUENCY}'
+  command: 'bash /clean-old-files.sh'
+  dockerargs: >-
+    --rm --name scheduler-job-clean_old_files
+    --volume ${COMPOSE_DIR}/optional-components/scheduler-job-clean_old_files/clean-old-files.sh:/clean-old-files.sh:ro
+    --env CLEAN_OLD_FILES_OPTIONS="${SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS}" ${SCHEDULER_JOB_CLEAN_OLD_FILES_VOLUMES}
+  image: '${SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE}'

birdhouse/optional-components/scheduler-job-clean_old_files/config/scheduler/docker-compose-extra.yml

@@ -0,0 +1,4 @@
+services:
+  scheduler:
+    volumes: 
+      - ./optional-components/scheduler-job-clean_old_files/config.yml:/scheduler-job-configs/clean_old_files.yml:ro

birdhouse/optional-components/scheduler-job-clean_old_files/default.env

@@ -0,0 +1,24 @@
+export SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE='${BASH_IMAGE}'
+
+export SCHEDULER_JOB_CLEAN_OLD_FILES_FREQUENCY="5 2 * * 0" # weekly on Sunday at 2:05
+
+export SCHEDULER_JOB_CLEAN_OLD_FILES_VOLUMES='$(for opt in ${SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS}; do printf " --volume %s:rw " "$(echo $opt | cut -d\| -f 1)"; done)'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE
+  SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS
+  SCHEDULER_JOB_CLEAN_OLD_FILES_VOLUMES
+"
+
+OPTIONAL_VARS="
+  $OPTIONAL_VARS
+  \$SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS
+  \$SCHEDULER_JOB_CLEAN_OLD_FILES_VOLUMES
+"
+
+VARS="
+  $VARS
+  \$SCHEDULER_JOB_CLEAN_OLD_FILES_IMAGE
+  \$SCHEDULER_JOB_CLEAN_OLD_FILES_FREQUENCY
+"

birdhouse/optional-components/scheduler-job-clean_old_files/pre-docker-compose-up.include

@@ -0,0 +1,3 @@
+if [ -z "$SCHEDULER_JOB_CLEAN_OLD_FILES_OPTIONS" ]; then
+  log WARN 'The scheduler-job-clean_old_files component is enabled but no files are scheduled to be deleted. Please reconfigure this component or disable it.'
+fi