Create template component for data deploy jobs

CHANGES.md

@@ -15,7 +15,30 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Create template component for data deploy jobs
+
+  New data deploy scheduler jobs no longer need to copy/paste lots of boilerplate code to create a new job.  
+  Instead they can simply define specific environment variables and then the can now use the 
+  `optional-components/scheduler-job-deploy_data` job will automatically generate a new data deploy job.
+
+  For example, if `XXXX` is added to the `SCHEDULER_JOB_DEPLOY_DATA_JOB_IDS` variable and the following 
+  variables are defined:
+
+  - `SCHEDULER_JOB_XXXX_DEPLOY_DATA_JOB_NAME`
+  - `SCHEDULER_JOB_XXXX_DEPLOY_DATA_JOB_COMMENT`
+  - `SCHEDULER_JOB_XXXX_DEPLOY_DATA_JOB_CHECKOUT_CACHE`
+  - `SCHEDULER_JOB_XXXX_DEPLOY_DATA_JOB_LOG_FILENAME`
+  - `SCHEDULER_JOB_XXXX_DEPLOY_DATA_JOB_SCHEDULE`
+  - `SCHEDULER_JOB_XXXX_DEPLOY_DATA_JOB_CONFIG_FILE`
+
+  a deploy data job will be automatically created.
+
+  See `optional-components/scheduler-job-deploy_raven_testdata/default.env` and 
+  `optional-components/scheduler-job-deploy_raven_testdata/default.env` for examples.
+
+  See `birdhouse/deployment/deploy-data` for details on how the deploy data job works.
 
 [2.15.0](https://github.com/bird-house/birdhouse-deploy/tree/2.15.0) (2025-05-27)
 ------------------------------------------------------------------------------------------------------------------

birdhouse/optional-components/scheduler-job-deploy_data/config/scheduler/docker-compose-extra.yml

@@ -0,0 +1,4 @@
+services:
+  scheduler:
+    volumes: 
+      - ./optional-components/scheduler-job-deploy_data/config.yml:/scheduler-job-configs/deploy_data.yml:ro

birdhouse/optional-components/scheduler-job-deploy_data/default.env

@@ -0,0 +1,11 @@
+export SCHEDULER_JOB_DEPLOY_DATA_JOB_DOCKER='docker'
+export SCHEDULER_JOB_DEPLOY_DATA_JOB_VERSION='19.03.6-git'
+export SCHEDULER_JOB_DEPLOY_DATA_JOB_IMAGE='${SCHEDULER_JOB_DEPLOY_DATA_JOB_DOCKER}:${SCHEDULER_JOB_DEPLOY_DATA_JOB_VERSION}'
+
+export SCHEDULER_JOB_DEPLOY_EXTRA_DOCKER_ARGS='$([ -n "$DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE" ] && echo "--volume ${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE}:${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE}:ro --env DEPLOY_DATA_GIT_SSH_IDENTITY_FILE=${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE} ")'
+
+DELAYED_EVAL="
+  $DELAYED_EVAL
+  SCHEDULER_JOB_DEPLOY_EXTRA_DOCKER_ARGS
+  SCHEDULER_JOB_DEPLOY_DATA_JOB_IMAGE
+"

birdhouse/optional-components/scheduler-job-deploy_data/pre-docker-compose-up.include

@@ -0,0 +1,52 @@
+generate_scheduler_job_deploy_data_template() {
+  job_id="$1"
+  name="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_NAME\"")"
+  comment="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_COMMENT\"")"
+  schedule="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_SCHEDULE\"")"
+  config_file="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_CONFIG_FILE\"")"
+  checkout_cache="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_CHECKOUT_CACHE\"")"
+  log_file_name="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_LOG_FILENAME\"")"
+  image="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_DOCKER_IMAGE\"")"
+  git_ssh_id_file="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_GIT_SSH_ID_FILE\"")"
+  extra_options="$(eval "echo \"\$SCHEDULER_JOB_${job_id}_DEPLOY_DATA_JOB_EXTRA_OPTIONS\"")"
+
+  error_msg = "No XXX found for deploy data job with id '$job_id'"
+  [ -z "$name" ] && log ERROR "$(sed $error_msg | 's/XXX/name/')" && return 1
+  [ -z "$schedule" ] && log ERROR "$(sed $error_msg | 's/XXX/schedule/')" && return 1
+  [ -z "$config_file" ] && log ERROR "$(sed $error_msg | 's/XXX/config_file/')" && return 1
+
+  comment="${comment:-"${name}"}"
+  checkout_cache="${checkout_cache:-"${name}"}"
+  image="${image:-"${SCHEDULER_JOB_DEPLOY_DATA_JOB_IMAGE}"}"
+  git_ssh_id_file="${git_ssh_id_file:-"${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE}"}"
+  log_file_name="${log_file_name:-"${name}.log"}"
+
+  if [ -n "$git_ssh_id_file" ]; then
+    git_ssh_id_options="--volume ${git_ssh_id_file}:${git_ssh_id_file}:ro --env DEPLOY_DATA_GIT_SSH_IDENTITY_FILE=${git_ssh_id_file} "
+  fi
+
+  echo "
+- name: '${name}'
+  comment: '${comment}'
+  schedule: '${schedule}'
+  command: '/deploy-data ${config_file}'
+  dockerargs: >-
+    --rm --name '${name}'
+    --volume /var/run/docker.sock:/var/run/docker.sock:ro
+    --volume ${COMPOSE_DIR}/deployment/deploy-data:/deploy-data:ro
+    --volume ${config_file}:${config_file}:ro
+    --volume ${checkout_cache}:${checkout_cache}:rw
+    --volume ${BIRDHOUSE_LOG_DIR}:${BIRDHOUSE_LOG_DIR}:rw
+    --env DEPLOY_DATA_CHECKOUT_CACHE=${checkout_cache}
+    --env DEPLOY_DATA_LOGFILE=${log_file_name} ${git_ssh_id_options} ${extra_options}
+  image: '${image}'
+"
+}
+
+SCHEDULER_JOB_DEPLOY_DATA_JOB_CONFIG_FILE="${COMPOSE_DIR}/optional-components/scheduler-job-deploy_data/config.yml"
+
+echo > "$SCHEDULER_JOB_DEPLOY_DATA_JOB_CONFIG_FILE"
+
+for job_id in $SCHEDULER_JOB_DEPLOY_DATA_JOB_IDS; do
+  generate_scheduler_job_deploy_data_template "$job_id" >> "$SCHEDULER_JOB_DEPLOY_DATA_JOB_CONFIG_FILE"
+done

birdhouse/optional-components/scheduler-job-deploy_raven_testdata/default.env

@@ -1,29 +1,20 @@
+export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_NAME=deploy_raven_testdata_to_thredds
+export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_COMMENT="Auto-deploy Raven testdata to Thredds for Raven tutorial notebooks."
 export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_CHECKOUT_CACHE='${BIRDHOUSE_DATA_PERSIST_ROOT}/deploy_data_cache/deploy_raven_testdata_to_thredds'
 export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_LOG_FILENAME='deploy_raven_testdata_to_thredds.log'
-export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_SCHEDULE="*/30 * * * *"  # UTC
+export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_SCHEDULE='*/30 * * * *'  # UTC
+export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_CONFIG_FILE="${COMPOSE_DIR}/deployment/deploy-data-raven-testdata-to-thredds.yml"
 
-export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_DOCKER='docker'
-export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_VERSION='19.03.6-git'
-export SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_IMAGE='${SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_DOCKER}:${SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_VERSION}'
-
-export SCHEDULER_JOB_RAVEN_DEPLOY_EXTRA_DOCKER_ARGS='$([ -n "$DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE" ] && echo "--volume ${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE}:${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE}:ro --env DEPLOY_DATA_GIT_SSH_IDENTITY_FILE=${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE} ")'
+export SCHEDULER_JOB_DEPLOY_DATA_JOB_IDS="
+  $SCHEDULER_JOB_DEPLOY_DATA_JOB_IDS
+  RAVEN
+"
 
 DELAYED_EVAL="
   $DELAYED_EVAL
   SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_CHECKOUT_CACHE
-  SCHEDULER_JOB_RAVEN_DEPLOY_EXTRA_DOCKER_ARGS
-  SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_IMAGE
-"
-
-VARS="
-  $VARS
-  \$SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_CHECKOUT_CACHE
-  \$SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_SCHEDULE$
-  \$SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_IMAGE
-  \$SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_LOG_FILENAME
 "
 
-OPTIONAL_VARS="
-  $OPTIONAL_VARS
-  \$SCHEDULER_JOB_RAVEN_DEPLOY_EXTRA_DOCKER_ARGS
+COMPONENT_DEPENDENCIES="
+  ./optional-components/scheduler-job-deploy_data
 "

birdhouse/optional-components/scheduler-job-deploy_raven_testdata/pre-docker-compose-up.include

@@ -1,7 +1,3 @@
 if ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'thredds[[:space:]]*$'; then
   log WARN 'The scheduler-job-deploy_raven_testdata component is enabled but the thredds component is not. This WILL cause problems. Please disable the scheduler-job-deploy_raven_testdata component.'
 fi
-
-if ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'raven[[:space:]]*$'; then
-  log WARN 'The scheduler-job-deploy_raven_testdata component is enabled but the raven component is not. Are you sure you want to enable the scheduler-job-deploy_raven_testdata component?'
-fi

birdhouse/optional-components/scheduler-job-deploy_xclim_testdata/default.env

@@ -1,29 +1,20 @@
+export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_NAME=deploy_xclim_testdata_to_thredds
+export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_COMMENT='Auto-deploy Xclim testdata to Thredds for Finch and Xclim tutorial notebooks.'
 export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_CHECKOUT_CACHE='${BIRDHOUSE_DATA_PERSIST_ROOT}/deploy_data_cache/deploy_xclim_testdata_to_thredds'
 export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_LOG_FILENAME='deploy_xclim_testdata_to_thredds.log'
-export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_SCHEDULE="7,37 * * * *"  # UTC
+export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_SCHEDULE='7,37 * * * *'  # UTC
+export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_CONFIG_FILE="${COMPOSE_DIR}/deployment/deploy-data-xclim-testdata-to-thredds.yml"
 
-export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_DOCKER='docker'
-export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_VERSION='19.03.6-git'
-export SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_IMAGE='${SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_DOCKER}:${SCHEDULER_JOB_RAVEN_DEPLOY_DATA_JOB_VERSION}'
-
-export SCHEDULER_JOB_XCLIM_DEPLOY_EXTRA_DOCKER_ARGS='$([ -n "$DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE" ] && echo "--volume ${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE}:${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE}:ro --env DEPLOY_DATA_GIT_SSH_IDENTITY_FILE=${DEPLOY_DATA_JOB_GIT_SSH_IDENTITY_FILE} ")'
+export SCHEDULER_JOB_DEPLOY_DATA_JOB_IDS="
+  $SCHEDULER_JOB_DEPLOY_DATA_JOB_IDS
+  XCLIM
+"
 
 DELAYED_EVAL="
   $DELAYED_EVAL
   SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_CHECKOUT_CACHE
-  SCHEDULER_JOB_XCLIM_DEPLOY_EXTRA_DOCKER_ARGS
-  SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_IMAGE
 "
 
-VARS="
-  $VARS
-  \$SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_CHECKOUT_CACHE
-  \$SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_SCHEDULE
-  \$SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_LOG_FILENAME
-  \$SCHEDULER_JOB_XCLIM_DEPLOY_DATA_JOB_IMAGE
-"
-
-OPTIONAL_VARS="
-  $OPTIONAL_VARS
-  \$SCHEDULER_JOB_XCLIM_DEPLOY_EXTRA_DOCKER_ARGS
-"
+COMPONENT_DEPENDENCIES="
+  ./optional-components/scheduler-job-deploy_data
+"