Add script that automatically updates postgres databases to a later version

CHANGES.md

@@ -15,7 +15,36 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Add script that automatically updates postgres databases to a later version
+
+  In anticipation of upgrading postgres databases in the future, this introduces a script that automatically
+  upgrades postgres databases using the backup/restore process.
+
+  This includes magpie and all WPS birds that use the postgres component. This does not include test component 
+  like `optional-components/generic_bird` and will not update custom components (ones not from this repository).
+
+  Test components are not assumed to have persistent data that needs to be updated and we cannot guarantee that
+  other postgres databases used by components outside this repository do not require additional steps (data
+  migrations) in order to comply with a different version of postgres. 
+
+  It will update postgres databases to the version specified by the `POSTGRES_VERSION_UPDATE` environment variable.
+  All of the old database files will be copied to a temporary directory in case you want to inspect them or revert 
+  this operation later on. To specify which directory to write these backups to set the `BIRDHOUSE_BACKUP_DATA_DIR` variable 
+  (default: `${TMPDIR:-/tmp}/birdhouse-postgres-migrate-backup/`)
+
+  Note that backups in the form of database dumps will also be written to the named volume or directory specified 
+  by the `BIRDHOUSE_BACKUP_VOLUME` variable.
+
+  For example, to update the current postgres databases to version 18.1 and write backups to `/tmp/test/`
+
+  ```sh
+  $ POSTGRES_VERSION_UPDATE=18.1 BIRDHOUSE_BACKUP_DATA_DIR=/tmp/test/ birdhouse/scripts/update-postgresh.sh
+  ```
+
+  In a future update we can update the postgres versions and tell users to run this script first in order to safely
+  migrate data from one version to the next.
 
 [2.20.3](https://github.com/bird-house/birdhouse-deploy/tree/2.20.3) (2026-01-13)
 ------------------------------------------------------------------------------------------------------------------

birdhouse/components/magpie/default.env

@@ -9,6 +9,10 @@ export MAGPIE_VERSION=4.2.0
 export MAGPIE_IMAGE='pavics/magpie:${MAGPIE_VERSION}'
 export MAGPIE_IMAGE_URI='registry.hub.docker.com/${MAGPIE_IMAGE}'
 
+export MAGPIE_POSTGRES_VERSION="9.6"
+export MAGPIE_POSTGRES_DOCKER=postgres
+export MAGPIE_POSTGRES_IMAGE='${MAGPIE_POSTGRES_DOCKER}:${MAGPIE_POSTGRES_VERSION}'
+
 export MAGPIE_DB_NAME="magpiedb"
 
 export MAGPIE_PERSIST_DIR='${BIRDHOUSE_DATA_PERSIST_ROOT}/magpie_persist'
@@ -35,6 +39,7 @@ export DELAYED_EVAL="
   MAGPIE_PERSIST_DIR
   MAGPIE_IMAGE
   MAGPIE_IMAGE_URI
+  MAGPIE_POSTGRES_IMAGE
 "
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here

birdhouse/components/magpie/docker-compose-extra.yml

@@ -43,7 +43,7 @@ services:
       start_interval: 5s
 
   postgres-magpie:
-    image: postgres:9.6
+    image: ${MAGPIE_POSTGRES_IMAGE}
     container_name: postgres-magpie
     # ports:  # should not be exposed directly, enable 'optional-components/database-external-ports' as needed
     env_file:

birdhouse/components/postgres/default.env

@@ -4,6 +4,10 @@
 # must use single quotes to avoid early expansion before overrides in env.local
 # are applied and must be added to the list of DELAYED_EVAL.
 
+export POSTGRES_VERSION="9.6"
+export POSTGRES_DOCKER=postgres
+export POSTGRES_IMAGE='${POSTGRES_DOCKER}:${POSTGRES_VERSION}'
+
 export POSTGRES_DATA_DIR='${BIRDHOUSE_DATA_PERSIST_ROOT}/frontend_persist'
 export __DEFAULT_BIRDHOUSE_POSTGRES_DB=birdhouse
 export BIRDHOUSE_POSTGRES_DB='${__DEFAULT_BIRDHOUSE_POSTGRES_DB}'
@@ -12,6 +16,7 @@ export DELAYED_EVAL="
   $DELAYED_EVAL
   POSTGRES_DATA_DIR
   BIRDHOUSE_POSTGRES_DB
+  POSTGRES_IMAGE
 "
 
 # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here

birdhouse/components/postgres/docker-compose-extra.yml

@@ -8,7 +8,7 @@ x-logging:
 
 services:
   postgres:
-    image: postgres:9.6
+    image: ${POSTGRES_IMAGE}
     container_name: postgres
     # ports:  # should not be exposed directly, enable 'optional-components/database-external-ports' as needed
     volumes:

birdhouse/scripts/update-postgres.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env sh
+
+# This script updates all postgres databases that are used by components in this repository.
+# This includes magpie and all WPS birds that use the postgres component.
+# This does not include test component like optional-components/generic_bird and will not update
+# custom components (ones not from this repository).
+#
+# It will update postgres databases to the version specified by the POSTGRES_VERSION_UPDATE
+# environment variable.
+# All of the old database files will be copied to a temporary directory in case you want to inspect
+# them or revert this operation later on. To specify which directory to write these backups to 
+# set the BIRDHOUSE_BACKUP_DATA_DIR variable (default: ${TMPDIR:-/tmp}/birdhouse-postgres-migrate-backup/)
+# Note that backups in the form of database dumps will also be written to the named volume or directory
+# specified by the BIRDHOUSE_BACKUP_VOLUME variable.
+# 
+# For example, to update the current postgres databases to version 18.1 and write backups to /tmp/test/
+# 
+# $ POSTGRES_VERSION_UPDATE=18.1 BIRDHOUSE_BACKUP_DATA_DIR=/tmp/test/ ./update-postgresh.sh
+#
+# This script will not update other postgres databases that may be present in this deployment but it can
+# be used as a starting point to figure out how to update other databases. For example, it may be possible
+# to update the postgres database used by generic_bird with (untested, please proceed with caution):
+#
+# $ birdhouse backup create --no-restic -a generic_bird
+# $ birdhouse compose down
+# $ ${THIS_DIR}/backup-datavolume.sh birdhouse_postgres_generic_bird "${BIRDHOUSE_BACKUP_DATA_DIR}"
+# $ docker volume rm birdhouse_postgres_generic_bird
+# $ GENERIC_BIRD_POSTGRES_IMAGE=postgres:18.1 birdhouse compose up -d
+# $ birdhouse backup restore --no-restic -a generic_bird
+#
+
+THIS_FILE="$(readlink -f "$0" || realpath "$0")"
+THIS_DIR="$(dirname "${THIS_FILE}")"
+BIRDHOUSE_EXE="${THIS_DIR}/../../bin/birdhouse"
+
+# load logging functions
+eval $(${BIRDHOUSE_EXE} configs --print-log-command)
+
+: ${POSTGRES_VERSION_UPDATE:?"$(log ERROR "POSTGRES_VERSION_UPDATE must be set")"}
+
+BIRDHOUSE_BACKUP_DATA_DIR="${BIRDHOUSE_BACKUP_DATA_DIR:-"${TMPDIR:-/tmp}"/birdhouse-postgres-migrate-backup/}"
+POSTGRES_COMPONENTS="-a magpie -a $(birdhouse -q configs -c 'echo $POSTGRES_DATABASES_TO_CREATE' | sed 's/ / -a /g')"
+
+log INFO "Migrating postgres databases to version ${POSTGRES_VERSION_UPDATE}"
+log INFO "Postgres databases for the following components will be updated: $(echo "${POSTGRES_COMPONENTS}" | sed 's/[[:space:]]*-a[[:space:]]*/ /g')"
+log INFO "Current databases files will be backed up to the directory: ${BIRDHOUSE_BACKUP_DATA_DIR}"
+
+log INFO "Starting the birdhouse stack in order to backup existing databases."
+${BIRDHOUSE_EXE} compose up -d
+
+log INFO "Backing up postgres databases from the following components: $(echo "${POSTGRES_COMPONENTS}" | sed 's/[[:space:]]*-a[[:space:]]*/ /g')"
+${BIRDHOUSE_EXE} backup create --no-restic ${POSTGRES_COMPONENTS}
+
+log INFO "Stopping the birdhouse stack in order to update postgres version"
+${BIRDHOUSE_EXE} compose down
+
+log INFO "Backing up all postgres data to ${BIRDHOUSE_BACKUP_DATA_DIR}."
+eval "$(${BIRDHOUSE_EXE} -q configs -c 'echo "MAGPIE_PERSIST_DIR=$MAGPIE_PERSIST_DIR; POSTGRES_DATA_DIR=$POSTGRES_DATA_DIR"')"
+
+mkdir -p ${BIRDHOUSE_BACKUP_DATA_DIR}
+mv "${MAGPIE_PERSIST_DIR}" "${BIRDHOUSE_BACKUP_DATA_DIR}"
+mv "${POSTGRES_DATA_DIR}" "${BIRDHOUSE_BACKUP_DATA_DIR}"
+
+MAGPIE_POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE} POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE} ${BIRDHOUSE_EXE} compose up -d
+
+${BIRDHOUSE_EXE} backup restore --no-restic ${POSTGRES_COMPONENTS}
+
+log WARN "Migration is now complete. Please ensure that the data has been upgraded properly.
+If you are satisfied that the databases have been updated properly please add the following to your local environment file:
+
+export MAGPIE_POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE} 
+export POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE}
+
+If you are not satified that the databases have been updated properly and you wish to revert these changes, you can do so by running:
+
+${BIRDHOUSE_EXE} compose down
+rm -r "${MAGPIE_PERSIST_DIR}"
+rm -r "${POSTGRES_DATA_DIR}"
+mv "${BIRDHOUSE_BACKUP_DATA_DIR}/$(basename "${MAGPIE_PERSIST_DIR}")" "${MAGPIE_PERSIST_DIR}" 
+mv "${BIRDHOUSE_BACKUP_DATA_DIR}/$(basename "${POSTGRES_DATA_DIR}")" "${POSTGRES_DATA_DIR}" 
+${BIRDHOUSE_EXE} compose up -d
+"