diff --git a/CHANGES.md b/CHANGES.md index 6d08822e4..4455b5ed8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,36 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes + +- Add script that automatically updates postgres databases to a later version + + In anticipation of upgrading postgres databases in the future, this introduces a script that automatically + upgrades postgres databases using the backup/restore process. + + This includes magpie and all WPS birds that use the postgres component. This does not include test component + like `optional-components/generic_bird` and will not update custom components (ones not from this repository). + + Test components are not assumed to have persistent data that needs to be updated and we cannot guarantee that + other postgres databases used by components outside this repository do not require additional steps (data + migrations) in order to comply with a different version of postgres. + + It will update postgres databases to the version specified by the `POSTGRES_VERSION_UPDATE` environment variable. + All of the old database files will be copied to a temporary directory in case you want to inspect them or revert + this operation later on. To specify which directory to write these backups to set the `BIRDHOUSE_BACKUP_DATA_DIR` variable + (default: `${TMPDIR:-/tmp}/birdhouse-postgres-migrate-backup/`) + + Note that backups in the form of database dumps will also be written to the named volume or directory specified + by the `BIRDHOUSE_BACKUP_VOLUME` variable. + + For example, to update the current postgres databases to version 18.1 and write backups to `/tmp/test/` + + ```sh + $ POSTGRES_VERSION_UPDATE=18.1 BIRDHOUSE_BACKUP_DATA_DIR=/tmp/test/ birdhouse/scripts/update-postgresh.sh + ``` + + In a future update we can update the postgres versions and tell users to run this script first in order to safely + migrate data from one version to the next. [2.20.3](https://github.com/bird-house/birdhouse-deploy/tree/2.20.3) (2026-01-13) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/magpie/default.env b/birdhouse/components/magpie/default.env index 9119dd643..f56a79c50 100644 --- a/birdhouse/components/magpie/default.env +++ b/birdhouse/components/magpie/default.env @@ -9,6 +9,10 @@ export MAGPIE_VERSION=4.2.0 export MAGPIE_IMAGE='pavics/magpie:${MAGPIE_VERSION}' export MAGPIE_IMAGE_URI='registry.hub.docker.com/${MAGPIE_IMAGE}' +export MAGPIE_POSTGRES_VERSION="9.6" +export MAGPIE_POSTGRES_DOCKER=postgres +export MAGPIE_POSTGRES_IMAGE='${MAGPIE_POSTGRES_DOCKER}:${MAGPIE_POSTGRES_VERSION}' + export MAGPIE_DB_NAME="magpiedb" export MAGPIE_PERSIST_DIR='${BIRDHOUSE_DATA_PERSIST_ROOT}/magpie_persist' @@ -35,6 +39,7 @@ export DELAYED_EVAL=" MAGPIE_PERSIST_DIR MAGPIE_IMAGE MAGPIE_IMAGE_URI + MAGPIE_POSTGRES_IMAGE " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here diff --git a/birdhouse/components/magpie/docker-compose-extra.yml b/birdhouse/components/magpie/docker-compose-extra.yml index 95c6f817d..aaf613e53 100644 --- a/birdhouse/components/magpie/docker-compose-extra.yml +++ b/birdhouse/components/magpie/docker-compose-extra.yml @@ -43,7 +43,7 @@ services: start_interval: 5s postgres-magpie: - image: postgres:9.6 + image: ${MAGPIE_POSTGRES_IMAGE} container_name: postgres-magpie # ports: # should not be exposed directly, enable 'optional-components/database-external-ports' as needed env_file: diff --git a/birdhouse/components/postgres/default.env b/birdhouse/components/postgres/default.env index 741fb9b4b..edbe9fc63 100644 --- a/birdhouse/components/postgres/default.env +++ b/birdhouse/components/postgres/default.env @@ -4,6 +4,10 @@ # must use single quotes to avoid early expansion before overrides in env.local # are applied and must be added to the list of DELAYED_EVAL. +export POSTGRES_VERSION="9.6" +export POSTGRES_DOCKER=postgres +export POSTGRES_IMAGE='${POSTGRES_DOCKER}:${POSTGRES_VERSION}' + export POSTGRES_DATA_DIR='${BIRDHOUSE_DATA_PERSIST_ROOT}/frontend_persist' export __DEFAULT_BIRDHOUSE_POSTGRES_DB=birdhouse export BIRDHOUSE_POSTGRES_DB='${__DEFAULT_BIRDHOUSE_POSTGRES_DB}' @@ -12,6 +16,7 @@ export DELAYED_EVAL=" $DELAYED_EVAL POSTGRES_DATA_DIR BIRDHOUSE_POSTGRES_DB + POSTGRES_IMAGE " # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here diff --git a/birdhouse/components/postgres/docker-compose-extra.yml b/birdhouse/components/postgres/docker-compose-extra.yml index 92a27326b..e6e155346 100644 --- a/birdhouse/components/postgres/docker-compose-extra.yml +++ b/birdhouse/components/postgres/docker-compose-extra.yml @@ -8,7 +8,7 @@ x-logging: services: postgres: - image: postgres:9.6 + image: ${POSTGRES_IMAGE} container_name: postgres # ports: # should not be exposed directly, enable 'optional-components/database-external-ports' as needed volumes: diff --git a/birdhouse/scripts/update-postgres.sh b/birdhouse/scripts/update-postgres.sh new file mode 100755 index 000000000..64740730d --- /dev/null +++ b/birdhouse/scripts/update-postgres.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env sh + +# This script updates all postgres databases that are used by components in this repository. +# This includes magpie and all WPS birds that use the postgres component. +# This does not include test component like optional-components/generic_bird and will not update +# custom components (ones not from this repository). +# +# It will update postgres databases to the version specified by the POSTGRES_VERSION_UPDATE +# environment variable. +# All of the old database files will be copied to a temporary directory in case you want to inspect +# them or revert this operation later on. To specify which directory to write these backups to +# set the BIRDHOUSE_BACKUP_DATA_DIR variable (default: ${TMPDIR:-/tmp}/birdhouse-postgres-migrate-backup/) +# Note that backups in the form of database dumps will also be written to the named volume or directory +# specified by the BIRDHOUSE_BACKUP_VOLUME variable. +# +# For example, to update the current postgres databases to version 18.1 and write backups to /tmp/test/ +# +# $ POSTGRES_VERSION_UPDATE=18.1 BIRDHOUSE_BACKUP_DATA_DIR=/tmp/test/ ./update-postgresh.sh +# +# This script will not update other postgres databases that may be present in this deployment but it can +# be used as a starting point to figure out how to update other databases. For example, it may be possible +# to update the postgres database used by generic_bird with (untested, please proceed with caution): +# +# $ birdhouse backup create --no-restic -a generic_bird +# $ birdhouse compose down +# $ ${THIS_DIR}/backup-datavolume.sh birdhouse_postgres_generic_bird "${BIRDHOUSE_BACKUP_DATA_DIR}" +# $ docker volume rm birdhouse_postgres_generic_bird +# $ GENERIC_BIRD_POSTGRES_IMAGE=postgres:18.1 birdhouse compose up -d +# $ birdhouse backup restore --no-restic -a generic_bird +# + +THIS_FILE="$(readlink -f "$0" || realpath "$0")" +THIS_DIR="$(dirname "${THIS_FILE}")" +BIRDHOUSE_EXE="${THIS_DIR}/../../bin/birdhouse" + +# load logging functions +eval $(${BIRDHOUSE_EXE} configs --print-log-command) + +: ${POSTGRES_VERSION_UPDATE:?"$(log ERROR "POSTGRES_VERSION_UPDATE must be set")"} + +BIRDHOUSE_BACKUP_DATA_DIR="${BIRDHOUSE_BACKUP_DATA_DIR:-"${TMPDIR:-/tmp}"/birdhouse-postgres-migrate-backup/}" +POSTGRES_COMPONENTS="-a magpie -a $(birdhouse -q configs -c 'echo $POSTGRES_DATABASES_TO_CREATE' | sed 's/ / -a /g')" + +log INFO "Migrating postgres databases to version ${POSTGRES_VERSION_UPDATE}" +log INFO "Postgres databases for the following components will be updated: $(echo "${POSTGRES_COMPONENTS}" | sed 's/[[:space:]]*-a[[:space:]]*/ /g')" +log INFO "Current databases files will be backed up to the directory: ${BIRDHOUSE_BACKUP_DATA_DIR}" + +log INFO "Starting the birdhouse stack in order to backup existing databases." +${BIRDHOUSE_EXE} compose up -d + +log INFO "Backing up postgres databases from the following components: $(echo "${POSTGRES_COMPONENTS}" | sed 's/[[:space:]]*-a[[:space:]]*/ /g')" +${BIRDHOUSE_EXE} backup create --no-restic ${POSTGRES_COMPONENTS} + +log INFO "Stopping the birdhouse stack in order to update postgres version" +${BIRDHOUSE_EXE} compose down + +log INFO "Backing up all postgres data to ${BIRDHOUSE_BACKUP_DATA_DIR}." +eval "$(${BIRDHOUSE_EXE} -q configs -c 'echo "MAGPIE_PERSIST_DIR=$MAGPIE_PERSIST_DIR; POSTGRES_DATA_DIR=$POSTGRES_DATA_DIR"')" + +mkdir -p ${BIRDHOUSE_BACKUP_DATA_DIR} +mv "${MAGPIE_PERSIST_DIR}" "${BIRDHOUSE_BACKUP_DATA_DIR}" +mv "${POSTGRES_DATA_DIR}" "${BIRDHOUSE_BACKUP_DATA_DIR}" + +MAGPIE_POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE} POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE} ${BIRDHOUSE_EXE} compose up -d + +${BIRDHOUSE_EXE} backup restore --no-restic ${POSTGRES_COMPONENTS} + +log WARN "Migration is now complete. Please ensure that the data has been upgraded properly. +If you are satisfied that the databases have been updated properly please add the following to your local environment file: + +export MAGPIE_POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE} +export POSTGRES_VERSION=${POSTGRES_VERSION_UPDATE} + +If you are not satified that the databases have been updated properly and you wish to revert these changes, you can do so by running: + +${BIRDHOUSE_EXE} compose down +rm -r "${MAGPIE_PERSIST_DIR}" +rm -r "${POSTGRES_DATA_DIR}" +mv "${BIRDHOUSE_BACKUP_DATA_DIR}/$(basename "${MAGPIE_PERSIST_DIR}")" "${MAGPIE_PERSIST_DIR}" +mv "${BIRDHOUSE_BACKUP_DATA_DIR}/$(basename "${POSTGRES_DATA_DIR}")" "${POSTGRES_DATA_DIR}" +${BIRDHOUSE_EXE} compose up -d +"