From 7b64ed7ff57c6fbec59c4fb1be89b6155b432d63 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Wed, 8 Apr 2020 21:57:36 -0400 Subject: [PATCH 1/7] finch2: 2nd Finch as an optional component See README.md for more details. --- birdhouse/common.env | 2 ++ birdhouse/docker-compose.yml | 2 +- birdhouse/optional-components/README.md | 21 ++++++++++++++++ .../optional-components/finch2/.gitignore | 4 +++ .../finch2/docker-compose-extra.yml | 25 +++++++++++++++++++ .../finch2-magpie-permission.cfg.template | 15 +++++++++++ .../finch2-magpie-provider.cfg.template | 8 ++++++ .../finch2_canarie_api_monitoring.py.template | 14 +++++++++++ .../finch2/wps.cfg.template | 8 ++++++ 9 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 birdhouse/optional-components/finch2/.gitignore create mode 100644 birdhouse/optional-components/finch2/docker-compose-extra.yml create mode 100644 birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template create mode 100644 birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template create mode 100644 birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template create mode 100644 birdhouse/optional-components/finch2/wps.cfg.template diff --git a/birdhouse/common.env b/birdhouse/common.env index ddf5702c..18fb2258 100644 --- a/birdhouse/common.env +++ b/birdhouse/common.env @@ -3,6 +3,8 @@ # Jupyter single-user server image export DOCKER_NOTEBOOK_IMAGE="pavics/workflow-tests:200312" +export FINCH_IMAGE="birdhouse/finch:version-0.5.2" + # Folder on the host to persist Jupyter user data (noteboooks, HOME settings) export JUPYTERHUB_USER_DATA_DIR="/data/jupyterhub_user_data" diff --git a/birdhouse/docker-compose.yml b/birdhouse/docker-compose.yml index e32425bd..4a9b3866 100644 --- a/birdhouse/docker-compose.yml +++ b/birdhouse/docker-compose.yml @@ -151,7 +151,7 @@ services: restart: always finch: - image: birdhouse/finch:version-0.5.2 + image: ${FINCH_IMAGE} container_name: finch environment: HOSTNAME: $HOSTNAME diff --git a/birdhouse/optional-components/README.md b/birdhouse/optional-components/README.md index 77bf6907..6c296ad4 100644 --- a/birdhouse/optional-components/README.md +++ b/birdhouse/optional-components/README.md @@ -51,3 +51,24 @@ server. No Twitcher/Magpie access control, this Test Thredds is directly behind the Nginx proxy. + + +## A second Finch server for testing + +How to enable in `env.local` (a copy from +[`env.local.example`](../env.local.example)): + +* Add `./optional-components/finch2` to `EXTRA_CONF_DIRS`. + +Second Finch service will be available at `http://PAVICS_FQDN:8010/wps` +or `https://PAVICS_FQDN_PUBLIC/twitcher/ows/proxy/finch2` where `PAVICS_FQDN` and +`PAVICS_FQDN_PUBLIC` are defined in your `env.local`. + +Use same docker image as regular Finch. + +Use sqlite DB instead of Postgres like the regular Finch. + +Magpie will be automatically configured to give complete public anonymous +access for this second Finch. + +Canarie monitoring will also be automatically configured for this second Finch. diff --git a/birdhouse/optional-components/finch2/.gitignore b/birdhouse/optional-components/finch2/.gitignore new file mode 100644 index 00000000..678b4b31 --- /dev/null +++ b/birdhouse/optional-components/finch2/.gitignore @@ -0,0 +1,4 @@ +wps.cfg +finch2-magpie-permission.cfg +finch2-magpie-provider.cfg +finch2_canarie_api_monitoring.py diff --git a/birdhouse/optional-components/finch2/docker-compose-extra.yml b/birdhouse/optional-components/finch2/docker-compose-extra.yml new file mode 100644 index 00000000..403f6e6e --- /dev/null +++ b/birdhouse/optional-components/finch2/docker-compose-extra.yml @@ -0,0 +1,25 @@ +version: '2.1' +services: + finch2: + image: ${FINCH_IMAGE} + container_name: finch2 + environment: + HOSTNAME: $HOSTNAME + HTTP_PORT: 5000 + PYWPS_CFG: /wps.cfg + ports: + - "8010:5000" + volumes: + - ./optional-components/finch2/wps.cfg:/wps.cfg:ro + - wps_outputs:/data/wpsoutputs + - /tmp + restart: always + + magpie: + volumes: + - ./optional-components/finch2/finch2-magpie-permission.cfg:/opt/local/src/magpie/config/permissions/finch2-magpie-permission.cfg:ro + - ./optional-components/finch2/finch2-magpie-provider.cfg:/opt/local/src/magpie/config/providers/finch2-magpie-provider.cfg:ro + + proxy: + volumes: + - ./optional-components/finch2/finch2_canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/finch2_canarie_api_monitoring.py:ro diff --git a/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template b/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template new file mode 100644 index 00000000..eeb532c3 --- /dev/null +++ b/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template @@ -0,0 +1,15 @@ +permissions: + - service: finch2 + permission: getcapabilities + group: anonymous + action: create + + - service: finch2 + permission: describeprocess + group: anonymous + action: create + + - service: finch2 + permission: execute + group: anonymous + action: create diff --git a/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template b/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template new file mode 100644 index 00000000..29081715 --- /dev/null +++ b/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template @@ -0,0 +1,8 @@ +providers: + finch2: + url: http://${PAVICS_FQDN}:8010/wps + title: Finch2 + public: true + c4i: false + type: wps + sync_type: wps diff --git a/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template b/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template new file mode 100644 index 00000000..1df397c1 --- /dev/null +++ b/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template @@ -0,0 +1,14 @@ +SERVICES['node']['monitoring'].update({ + 'Finch2-public': { + 'request': { + 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/finch2?service=WPS&version=1.0.0&request=GetCapabilities' + }, + }, + 'Finch2': { + 'request': { + 'url': 'http://${PAVICS_FQDN}:8010/wps?service=WPS&version=1.0.0&request=GetCapabilities' + } + }, +}) + +# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python diff --git a/birdhouse/optional-components/finch2/wps.cfg.template b/birdhouse/optional-components/finch2/wps.cfg.template new file mode 100644 index 00000000..0ecc1f1c --- /dev/null +++ b/birdhouse/optional-components/finch2/wps.cfg.template @@ -0,0 +1,8 @@ +[server] +outputurl = https://${PAVICS_FQDN_PUBLIC}/wpsoutputs +outputpath = /data/wpsoutputs + +[logging] +level = DEBUG + +${EXTRA_PYWPS_CONFIG} From 2de75bc9551a141d38807a4bccf9bf12fbf0257d Mon Sep 17 00:00:00 2001 From: Long Vu Date: Wed, 8 Apr 2020 22:38:09 -0400 Subject: [PATCH 2/7] finch2: allow to customize docker image, service name and port Bonus, this change also prevent the same value to be hardcoded multiple times at various locations. Optional components should probably not hook directly into `common.env` and `pavics-compose.sh`. Another refactoring for another day. --- birdhouse/common.env | 3 +++ birdhouse/optional-components/README.md | 12 ++++++++---- .../finch2/docker-compose-extra.yml | 4 ++-- .../finch2/finch2-magpie-permission.cfg.template | 6 +++--- .../finch2/finch2-magpie-provider.cfg.template | 6 +++--- .../finch2/finch2_canarie_api_monitoring.py.template | 8 ++++---- birdhouse/pavics-compose.sh | 2 ++ 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/birdhouse/common.env b/birdhouse/common.env index 18fb2258..0cdbee5e 100644 --- a/birdhouse/common.env +++ b/birdhouse/common.env @@ -4,6 +4,9 @@ export DOCKER_NOTEBOOK_IMAGE="pavics/workflow-tests:200312" export FINCH_IMAGE="birdhouse/finch:version-0.5.2" +export FINCH2_IMAGE="$FINCH_IMAGE" +export FINCH2_PORT="8010" +export FINCH2_NAME="finch2" # Folder on the host to persist Jupyter user data (noteboooks, HOME settings) export JUPYTERHUB_USER_DATA_DIR="/data/jupyterhub_user_data" diff --git a/birdhouse/optional-components/README.md b/birdhouse/optional-components/README.md index 6c296ad4..3043e51d 100644 --- a/birdhouse/optional-components/README.md +++ b/birdhouse/optional-components/README.md @@ -60,11 +60,15 @@ How to enable in `env.local` (a copy from * Add `./optional-components/finch2` to `EXTRA_CONF_DIRS`. -Second Finch service will be available at `http://PAVICS_FQDN:8010/wps` -or `https://PAVICS_FQDN_PUBLIC/twitcher/ows/proxy/finch2` where `PAVICS_FQDN` and -`PAVICS_FQDN_PUBLIC` are defined in your `env.local`. +* Optionally set `FINCH2_IMAGE`, `FINCH2_PORT`, `FINCH2_NAME` in `env.local` + for further customizations. Default values are in [`common.env`](../common.env). + +Second Finch service will be available at `http://PAVICS_FQDN:FINCH2_PORT/wps` +or `https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/FINCH2_NAME` where +`PAVICS_FQDN`, `PAVICS_FQDN_PUBLIC` and `TWITCHER_PROTECTED_PATH` are defined +in your `env.local`. -Use same docker image as regular Finch. +Use same docker image as regular Finch by default but can be customized. Use sqlite DB instead of Postgres like the regular Finch. diff --git a/birdhouse/optional-components/finch2/docker-compose-extra.yml b/birdhouse/optional-components/finch2/docker-compose-extra.yml index 403f6e6e..08613400 100644 --- a/birdhouse/optional-components/finch2/docker-compose-extra.yml +++ b/birdhouse/optional-components/finch2/docker-compose-extra.yml @@ -1,14 +1,14 @@ version: '2.1' services: finch2: - image: ${FINCH_IMAGE} + image: ${FINCH2_IMAGE} container_name: finch2 environment: HOSTNAME: $HOSTNAME HTTP_PORT: 5000 PYWPS_CFG: /wps.cfg ports: - - "8010:5000" + - "${FINCH2_PORT}:5000" volumes: - ./optional-components/finch2/wps.cfg:/wps.cfg:ro - wps_outputs:/data/wpsoutputs diff --git a/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template b/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template index eeb532c3..849b1c26 100644 --- a/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template +++ b/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template @@ -1,15 +1,15 @@ permissions: - - service: finch2 + - service: ${FINCH2_NAME} permission: getcapabilities group: anonymous action: create - - service: finch2 + - service: ${FINCH2_NAME} permission: describeprocess group: anonymous action: create - - service: finch2 + - service: ${FINCH2_NAME} permission: execute group: anonymous action: create diff --git a/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template b/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template index 29081715..ce084660 100644 --- a/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template +++ b/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template @@ -1,7 +1,7 @@ providers: - finch2: - url: http://${PAVICS_FQDN}:8010/wps - title: Finch2 + ${FINCH2_NAME}: + url: http://${PAVICS_FQDN}:${FINCH2_PORT}/wps + title: ${FINCH2_NAME} public: true c4i: false type: wps diff --git a/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template b/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template index 1df397c1..300cc94c 100644 --- a/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template @@ -1,12 +1,12 @@ SERVICES['node']['monitoring'].update({ - 'Finch2-public': { + '${FINCH2_NAME}-public': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/finch2?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/${FINCH2_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' }, }, - 'Finch2': { + '${FINCH2_NAME}': { 'request': { - 'url': 'http://${PAVICS_FQDN}:8010/wps?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': 'http://${PAVICS_FQDN}:${FINCH2_PORT}/wps?service=WPS&version=1.0.0&request=GetCapabilities' } }, }) diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index bd1615af..1c14dd6a 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -42,6 +42,8 @@ OPTIONAL_VARS=' $JUPYTER_DEMO_USER $JUPYTER_LOGIN_BANNER_TOP_SECTION $JUPYTER_LOGIN_BANNER_BOTTOM_SECTION + $FINCH2_PORT + $FINCH2_NAME ' # we switch to the real directory of the script, so it still works when used from $PATH From 959344c4478f0eafaec980716459e31097c1da58 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 9 Apr 2020 13:04:04 -0400 Subject: [PATCH 3/7] finch2: add Postgres DB for proper WPS logging of Async WPS requests Default was SQLite in-memory and it does not clear completed async wps requests so we quickly run into the default 2 max concurrent wps requests ! Use a separate DB so this optional component do not leave traces when we disable it. This will also allow us to test new different version of Postgres DB if needed. Default version is latest Postgres, not the old version currently in used. --- birdhouse/optional-components/README.md | 4 +++- .../finch2/docker-compose-extra.yml | 17 +++++++++++++++++ .../finch2/postgres/.gitignore | 1 + .../finch2/postgres/credentials.env.template | 3 +++ .../create-wps-databases.sql | 1 + .../optional-components/finch2/wps.cfg.template | 1 + 6 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 birdhouse/optional-components/finch2/postgres/.gitignore create mode 100644 birdhouse/optional-components/finch2/postgres/credentials.env.template create mode 100644 birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql diff --git a/birdhouse/optional-components/README.md b/birdhouse/optional-components/README.md index 3043e51d..4be33990 100644 --- a/birdhouse/optional-components/README.md +++ b/birdhouse/optional-components/README.md @@ -70,7 +70,9 @@ in your `env.local`. Use same docker image as regular Finch by default but can be customized. -Use sqlite DB instead of Postgres like the regular Finch. +Use a separate Postgres DB for this optional component to be completely +self-contained and to allow experimenting with different versions of Postgres +DB. Magpie will be automatically configured to give complete public anonymous access for this second Finch. diff --git a/birdhouse/optional-components/finch2/docker-compose-extra.yml b/birdhouse/optional-components/finch2/docker-compose-extra.yml index 08613400..a4ca8531 100644 --- a/birdhouse/optional-components/finch2/docker-compose-extra.yml +++ b/birdhouse/optional-components/finch2/docker-compose-extra.yml @@ -13,6 +13,20 @@ services: - ./optional-components/finch2/wps.cfg:/wps.cfg:ro - wps_outputs:/data/wpsoutputs - /tmp + links: + - postgres_finch2 + restart: always + + postgres_finch2: + image: ${FINCH2_POSTGRES_IMAGE:-postgres:10.12} + container_name: postgres_finch2 + volumes: + - postgres_finch2:/var/lib/postgresql/data/pgdata + - ./optional-components/finch2/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + environment: + PGDATA: /var/lib/postgresql/data/pgdata + env_file: + - ./config/postgres/credentials.env restart: always magpie: @@ -23,3 +37,6 @@ services: proxy: volumes: - ./optional-components/finch2/finch2_canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/finch2_canarie_api_monitoring.py:ro + +volumes: + postgres_finch2: {} diff --git a/birdhouse/optional-components/finch2/postgres/.gitignore b/birdhouse/optional-components/finch2/postgres/.gitignore new file mode 100644 index 00000000..983353b4 --- /dev/null +++ b/birdhouse/optional-components/finch2/postgres/.gitignore @@ -0,0 +1 @@ +credentials.env diff --git a/birdhouse/optional-components/finch2/postgres/credentials.env.template b/birdhouse/optional-components/finch2/postgres/credentials.env.template new file mode 100644 index 00000000..577d821c --- /dev/null +++ b/birdhouse/optional-components/finch2/postgres/credentials.env.template @@ -0,0 +1,3 @@ +POSTGRES_USER=${POSTGRES_PAVICS_USERNAME} +POSTGRES_PASSWORD=${POSTGRES_PAVICS_PASSWORD} +POSTGRES_DB=pavics diff --git a/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql b/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql new file mode 100644 index 00000000..63ab32a2 --- /dev/null +++ b/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql @@ -0,0 +1 @@ +CREATE DATABASE finch2; diff --git a/birdhouse/optional-components/finch2/wps.cfg.template b/birdhouse/optional-components/finch2/wps.cfg.template index 0ecc1f1c..16391cde 100644 --- a/birdhouse/optional-components/finch2/wps.cfg.template +++ b/birdhouse/optional-components/finch2/wps.cfg.template @@ -4,5 +4,6 @@ outputpath = /data/wpsoutputs [logging] level = DEBUG +database=postgresql://${POSTGRES_PAVICS_USERNAME}:${POSTGRES_PAVICS_PASSWORD}@postgres_finch2/finch2 ${EXTRA_PYWPS_CONFIG} From 4b22506b41c9827b720592e9bbaedf6c1a9a99e9 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 9 Apr 2020 13:09:53 -0400 Subject: [PATCH 4/7] clear-running-wps-jobs-in-db.sh: support different postgres container name --- birdhouse/scripts/clear-running-wps-jobs-in-db.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh index d4e6d462..ef57a105 100755 --- a/birdhouse/scripts/clear-running-wps-jobs-in-db.sh +++ b/birdhouse/scripts/clear-running-wps-jobs-in-db.sh @@ -15,8 +15,12 @@ else shift fi +if [ -z "$POSTGRES_CONTAINER_NAME" ]; then + POSTGRES_CONTAINER_NAME=postgres +fi + set -x -docker exec postgres psql -U $POSTGRES_USER $DB_NAME -c "select * from pywps_requests where percent_done > -1 and percent_done < 100.0;" +docker exec $POSTGRES_CONTAINER_NAME psql -U $POSTGRES_USER $DB_NAME -c "select * from pywps_requests where percent_done > -1 and percent_done < 100.0;" set +x echo " @@ -27,4 +31,4 @@ Clear those jobs? (Ctrl-C to cancel, any keys to continue)" read a set -x -docker exec postgres psql -U $POSTGRES_USER $DB_NAME -c "delete from pywps_requests where percent_done > -1 and percent_done < 100.0;" +docker exec $POSTGRES_CONTAINER_NAME psql -U $POSTGRES_USER $DB_NAME -c "delete from pywps_requests where percent_done > -1 and percent_done < 100.0;" From 12a381cd6408fa99498fd31697bde288c3be21c5 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 9 Apr 2020 13:43:01 -0400 Subject: [PATCH 5/7] finch2: make this component generic to accommodate any other WPS than Finch We can plug in other WPS docker images in this skeleton/generic component. All the elements that need to change are configurable (was just missing the FINCH2_INTERNAL_PORT). The HOSTNAME and HTTP_PORT env var are only for older birds so disabled them. Set WPS logging to a more suitable production logging level, can easily be changed later. --- birdhouse/optional-components/finch2/docker-compose-extra.yml | 4 +--- birdhouse/optional-components/finch2/wps.cfg.template | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/birdhouse/optional-components/finch2/docker-compose-extra.yml b/birdhouse/optional-components/finch2/docker-compose-extra.yml index a4ca8531..f067b1f8 100644 --- a/birdhouse/optional-components/finch2/docker-compose-extra.yml +++ b/birdhouse/optional-components/finch2/docker-compose-extra.yml @@ -4,11 +4,9 @@ services: image: ${FINCH2_IMAGE} container_name: finch2 environment: - HOSTNAME: $HOSTNAME - HTTP_PORT: 5000 PYWPS_CFG: /wps.cfg ports: - - "${FINCH2_PORT}:5000" + - "${FINCH2_PORT}:${FINCH2_INTERNAL_PORT:-5000}" volumes: - ./optional-components/finch2/wps.cfg:/wps.cfg:ro - wps_outputs:/data/wpsoutputs diff --git a/birdhouse/optional-components/finch2/wps.cfg.template b/birdhouse/optional-components/finch2/wps.cfg.template index 16391cde..2d657039 100644 --- a/birdhouse/optional-components/finch2/wps.cfg.template +++ b/birdhouse/optional-components/finch2/wps.cfg.template @@ -3,7 +3,7 @@ outputurl = https://${PAVICS_FQDN_PUBLIC}/wpsoutputs outputpath = /data/wpsoutputs [logging] -level = DEBUG +level = INFO database=postgresql://${POSTGRES_PAVICS_USERNAME}:${POSTGRES_PAVICS_PASSWORD}@postgres_finch2/finch2 ${EXTRA_PYWPS_CONFIG} From c265273b531b6da2a24c5896ba10fa6a3fc6bd0d Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 Apr 2020 22:59:35 -0400 Subject: [PATCH 6/7] optional-components: rename finch2 to generic_bird to make clear is is generic and not related to Finch --- birdhouse/common.env | 6 ++-- birdhouse/optional-components/README.md | 17 ++++++----- .../optional-components/finch2/.gitignore | 6 ++-- .../finch2/docker-compose-extra.yml | 30 +++++++++---------- .../finch2-magpie-permission.cfg.template | 6 ++-- .../finch2-magpie-provider.cfg.template | 6 ++-- .../finch2_canarie_api_monitoring.py.template | 8 ++--- .../create-wps-databases.sql | 2 +- .../finch2/wps.cfg.template | 2 +- birdhouse/pavics-compose.sh | 4 +-- 10 files changed, 45 insertions(+), 42 deletions(-) diff --git a/birdhouse/common.env b/birdhouse/common.env index 18c79716..56ad353e 100644 --- a/birdhouse/common.env +++ b/birdhouse/common.env @@ -4,9 +4,9 @@ export DOCKER_NOTEBOOK_IMAGE="pavics/workflow-tests:200312" export FINCH_IMAGE="birdhouse/finch:version-0.5.2" -export FINCH2_IMAGE="$FINCH_IMAGE" -export FINCH2_PORT="8010" -export FINCH2_NAME="finch2" +export GENERIC_BIRD_IMAGE="$FINCH_IMAGE" +export GENERIC_BIRD_PORT="8010" +export GENERIC_BIRD_NAME="generic_bird" # Folder on the host to persist Jupyter user data (noteboooks, HOME settings) export JUPYTERHUB_USER_DATA_DIR="/data/jupyterhub_user_data" diff --git a/birdhouse/optional-components/README.md b/birdhouse/optional-components/README.md index 4be33990..e1ab5b1a 100644 --- a/birdhouse/optional-components/README.md +++ b/birdhouse/optional-components/README.md @@ -53,18 +53,21 @@ No Twitcher/Magpie access control, this Test Thredds is directly behind the Nginx proxy. -## A second Finch server for testing +## A generic bird WPS service + +Can be used to quickly deploy any birds temporarily without changing code. +Good to preview new birds or test alternative configuration of existing birds. How to enable in `env.local` (a copy from [`env.local.example`](../env.local.example)): -* Add `./optional-components/finch2` to `EXTRA_CONF_DIRS`. +* Add `./optional-components/generic_bird` to `EXTRA_CONF_DIRS`. -* Optionally set `FINCH2_IMAGE`, `FINCH2_PORT`, `FINCH2_NAME` in `env.local` +* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`, `GENERIC_BIRD_NAME` in `env.local` for further customizations. Default values are in [`common.env`](../common.env). -Second Finch service will be available at `http://PAVICS_FQDN:FINCH2_PORT/wps` -or `https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/FINCH2_NAME` where +The WPS service will be available at `http://PAVICS_FQDN:GENERIC_BIRD_PORT/wps` +or `https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/GENERIC_BIRD_NAME` where `PAVICS_FQDN`, `PAVICS_FQDN_PUBLIC` and `TWITCHER_PROTECTED_PATH` are defined in your `env.local`. @@ -75,6 +78,6 @@ self-contained and to allow experimenting with different versions of Postgres DB. Magpie will be automatically configured to give complete public anonymous -access for this second Finch. +access for this WPS service. -Canarie monitoring will also be automatically configured for this second Finch. +Canarie monitoring will also be automatically configured for this WPS service. diff --git a/birdhouse/optional-components/finch2/.gitignore b/birdhouse/optional-components/finch2/.gitignore index 678b4b31..0170dff7 100644 --- a/birdhouse/optional-components/finch2/.gitignore +++ b/birdhouse/optional-components/finch2/.gitignore @@ -1,4 +1,4 @@ wps.cfg -finch2-magpie-permission.cfg -finch2-magpie-provider.cfg -finch2_canarie_api_monitoring.py +generic_bird-magpie-permission.cfg +generic_bird-magpie-provider.cfg +generic_bird_canarie_api_monitoring.py diff --git a/birdhouse/optional-components/finch2/docker-compose-extra.yml b/birdhouse/optional-components/finch2/docker-compose-extra.yml index f067b1f8..0db30924 100644 --- a/birdhouse/optional-components/finch2/docker-compose-extra.yml +++ b/birdhouse/optional-components/finch2/docker-compose-extra.yml @@ -1,26 +1,26 @@ version: '2.1' services: - finch2: - image: ${FINCH2_IMAGE} - container_name: finch2 + generic_bird: + image: ${GENERIC_BIRD_IMAGE} + container_name: generic_bird environment: PYWPS_CFG: /wps.cfg ports: - - "${FINCH2_PORT}:${FINCH2_INTERNAL_PORT:-5000}" + - "${GENERIC_BIRD_PORT}:${GENERIC_BIRD_INTERNAL_PORT:-5000}" volumes: - - ./optional-components/finch2/wps.cfg:/wps.cfg:ro + - ./optional-components/generic_bird/wps.cfg:/wps.cfg:ro - wps_outputs:/data/wpsoutputs - /tmp links: - - postgres_finch2 + - postgres_generic_bird restart: always - postgres_finch2: - image: ${FINCH2_POSTGRES_IMAGE:-postgres:10.12} - container_name: postgres_finch2 + postgres_generic_bird: + image: ${GENERIC_BIRD_POSTGRES_IMAGE:-postgres:10.12} + container_name: postgres_generic_bird volumes: - - postgres_finch2:/var/lib/postgresql/data/pgdata - - ./optional-components/finch2/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + - postgres_generic_bird:/var/lib/postgresql/data/pgdata + - ./optional-components/generic_bird/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d environment: PGDATA: /var/lib/postgresql/data/pgdata env_file: @@ -29,12 +29,12 @@ services: magpie: volumes: - - ./optional-components/finch2/finch2-magpie-permission.cfg:/opt/local/src/magpie/config/permissions/finch2-magpie-permission.cfg:ro - - ./optional-components/finch2/finch2-magpie-provider.cfg:/opt/local/src/magpie/config/providers/finch2-magpie-provider.cfg:ro + - ./optional-components/generic_bird/generic_bird-magpie-permission.cfg:/opt/local/src/magpie/config/permissions/generic_bird-magpie-permission.cfg:ro + - ./optional-components/generic_bird/generic_bird-magpie-provider.cfg:/opt/local/src/magpie/config/providers/generic_bird-magpie-provider.cfg:ro proxy: volumes: - - ./optional-components/finch2/finch2_canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/finch2_canarie_api_monitoring.py:ro + - ./optional-components/generic_bird/generic_bird_canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/generic_bird_canarie_api_monitoring.py:ro volumes: - postgres_finch2: {} + postgres_generic_bird: {} diff --git a/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template b/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template index 849b1c26..b0c060f2 100644 --- a/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template +++ b/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template @@ -1,15 +1,15 @@ permissions: - - service: ${FINCH2_NAME} + - service: ${GENERIC_BIRD_NAME} permission: getcapabilities group: anonymous action: create - - service: ${FINCH2_NAME} + - service: ${GENERIC_BIRD_NAME} permission: describeprocess group: anonymous action: create - - service: ${FINCH2_NAME} + - service: ${GENERIC_BIRD_NAME} permission: execute group: anonymous action: create diff --git a/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template b/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template index ce084660..0bb056b1 100644 --- a/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template +++ b/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template @@ -1,7 +1,7 @@ providers: - ${FINCH2_NAME}: - url: http://${PAVICS_FQDN}:${FINCH2_PORT}/wps - title: ${FINCH2_NAME} + ${GENERIC_BIRD_NAME}: + url: http://${PAVICS_FQDN}:${GENERIC_BIRD_PORT}/wps + title: ${GENERIC_BIRD_NAME} public: true c4i: false type: wps diff --git a/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template b/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template index 300cc94c..2e04b446 100644 --- a/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template +++ b/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template @@ -1,12 +1,12 @@ SERVICES['node']['monitoring'].update({ - '${FINCH2_NAME}-public': { + '${GENERIC_BIRD_NAME}-public': { 'request': { - 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/${FINCH2_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/${GENERIC_BIRD_NAME}?service=WPS&version=1.0.0&request=GetCapabilities' }, }, - '${FINCH2_NAME}': { + '${GENERIC_BIRD_NAME}': { 'request': { - 'url': 'http://${PAVICS_FQDN}:${FINCH2_PORT}/wps?service=WPS&version=1.0.0&request=GetCapabilities' + 'url': 'http://${PAVICS_FQDN}:${GENERIC_BIRD_PORT}/wps?service=WPS&version=1.0.0&request=GetCapabilities' } }, }) diff --git a/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql b/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql index 63ab32a2..7cebd314 100644 --- a/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql +++ b/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql @@ -1 +1 @@ -CREATE DATABASE finch2; +CREATE DATABASE generic_bird; diff --git a/birdhouse/optional-components/finch2/wps.cfg.template b/birdhouse/optional-components/finch2/wps.cfg.template index 2d657039..f23caabf 100644 --- a/birdhouse/optional-components/finch2/wps.cfg.template +++ b/birdhouse/optional-components/finch2/wps.cfg.template @@ -4,6 +4,6 @@ outputpath = /data/wpsoutputs [logging] level = INFO -database=postgresql://${POSTGRES_PAVICS_USERNAME}:${POSTGRES_PAVICS_PASSWORD}@postgres_finch2/finch2 +database=postgresql://${POSTGRES_PAVICS_USERNAME}:${POSTGRES_PAVICS_PASSWORD}@postgres_generic_bird/generic_bird ${EXTRA_PYWPS_CONFIG} diff --git a/birdhouse/pavics-compose.sh b/birdhouse/pavics-compose.sh index c81a07c4..006fa8d5 100755 --- a/birdhouse/pavics-compose.sh +++ b/birdhouse/pavics-compose.sh @@ -46,8 +46,8 @@ OPTIONAL_VARS=' $AUTODEPLOY_PLATFORM_FREQUENCY $AUTODEPLOY_NOTEBOOK_FREQUENCY $AUTODEPLOY_EXTRA_SCHEDULER_JOBS - $FINCH2_PORT - $FINCH2_NAME + $GENERIC_BIRD_PORT + $GENERIC_BIRD_NAME ' # we switch to the real directory of the script, so it still works when used from $PATH From 817c6f4485b9b0abdd5ecd09fb311a999659b44a Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 Apr 2020 23:02:33 -0400 Subject: [PATCH 7/7] optional-components: physical file rename finch2 to generic_bird to make clear is is generic and not related to Finch --- birdhouse/optional-components/{finch2 => generic_bird}/.gitignore | 0 .../{finch2 => generic_bird}/docker-compose-extra.yml | 0 .../generic_bird-magpie-permission.cfg.template} | 0 .../generic_bird-magpie-provider.cfg.template} | 0 .../generic_bird_canarie_api_monitoring.py.template} | 0 .../{finch2 => generic_bird}/postgres/.gitignore | 0 .../{finch2 => generic_bird}/postgres/credentials.env.template | 0 .../postgres/docker-entrypoint-initdb.d/create-wps-databases.sql | 0 .../optional-components/{finch2 => generic_bird}/wps.cfg.template | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename birdhouse/optional-components/{finch2 => generic_bird}/.gitignore (100%) rename birdhouse/optional-components/{finch2 => generic_bird}/docker-compose-extra.yml (100%) rename birdhouse/optional-components/{finch2/finch2-magpie-permission.cfg.template => generic_bird/generic_bird-magpie-permission.cfg.template} (100%) rename birdhouse/optional-components/{finch2/finch2-magpie-provider.cfg.template => generic_bird/generic_bird-magpie-provider.cfg.template} (100%) rename birdhouse/optional-components/{finch2/finch2_canarie_api_monitoring.py.template => generic_bird/generic_bird_canarie_api_monitoring.py.template} (100%) rename birdhouse/optional-components/{finch2 => generic_bird}/postgres/.gitignore (100%) rename birdhouse/optional-components/{finch2 => generic_bird}/postgres/credentials.env.template (100%) rename birdhouse/optional-components/{finch2 => generic_bird}/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql (100%) rename birdhouse/optional-components/{finch2 => generic_bird}/wps.cfg.template (100%) diff --git a/birdhouse/optional-components/finch2/.gitignore b/birdhouse/optional-components/generic_bird/.gitignore similarity index 100% rename from birdhouse/optional-components/finch2/.gitignore rename to birdhouse/optional-components/generic_bird/.gitignore diff --git a/birdhouse/optional-components/finch2/docker-compose-extra.yml b/birdhouse/optional-components/generic_bird/docker-compose-extra.yml similarity index 100% rename from birdhouse/optional-components/finch2/docker-compose-extra.yml rename to birdhouse/optional-components/generic_bird/docker-compose-extra.yml diff --git a/birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template b/birdhouse/optional-components/generic_bird/generic_bird-magpie-permission.cfg.template similarity index 100% rename from birdhouse/optional-components/finch2/finch2-magpie-permission.cfg.template rename to birdhouse/optional-components/generic_bird/generic_bird-magpie-permission.cfg.template diff --git a/birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template b/birdhouse/optional-components/generic_bird/generic_bird-magpie-provider.cfg.template similarity index 100% rename from birdhouse/optional-components/finch2/finch2-magpie-provider.cfg.template rename to birdhouse/optional-components/generic_bird/generic_bird-magpie-provider.cfg.template diff --git a/birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template b/birdhouse/optional-components/generic_bird/generic_bird_canarie_api_monitoring.py.template similarity index 100% rename from birdhouse/optional-components/finch2/finch2_canarie_api_monitoring.py.template rename to birdhouse/optional-components/generic_bird/generic_bird_canarie_api_monitoring.py.template diff --git a/birdhouse/optional-components/finch2/postgres/.gitignore b/birdhouse/optional-components/generic_bird/postgres/.gitignore similarity index 100% rename from birdhouse/optional-components/finch2/postgres/.gitignore rename to birdhouse/optional-components/generic_bird/postgres/.gitignore diff --git a/birdhouse/optional-components/finch2/postgres/credentials.env.template b/birdhouse/optional-components/generic_bird/postgres/credentials.env.template similarity index 100% rename from birdhouse/optional-components/finch2/postgres/credentials.env.template rename to birdhouse/optional-components/generic_bird/postgres/credentials.env.template diff --git a/birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql b/birdhouse/optional-components/generic_bird/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql similarity index 100% rename from birdhouse/optional-components/finch2/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql rename to birdhouse/optional-components/generic_bird/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql diff --git a/birdhouse/optional-components/finch2/wps.cfg.template b/birdhouse/optional-components/generic_bird/wps.cfg.template similarity index 100% rename from birdhouse/optional-components/finch2/wps.cfg.template rename to birdhouse/optional-components/generic_bird/wps.cfg.template