Merge pull request #41 from bird-house/add-finch2-optional-component

Add generic_bird optional component.

Could be used as work-around for issue bird-house/finch#121 so we have a Finch dedicated for async request only.

This component is generic and configurable enough it could even accommodate any WPS.  It has its own Postgres instance so we can use it later to experiment with different version of Postgres.

Test server: https://lvupavics-lvu.pagekite.me/twitcher/ows/proxy/finchasync, https://lvupavics-lvu.pagekite.me/canarie/node/service/status

See the `README.md` update for more info.

Note the docker image, service name and port is customizable (my test server do not use the default service name).

It starts out as a Finch2 but now its rather a generic skeleton to plugin any birds we need. I wish I had this when I was testing out the new Thunderbird from PCIC.

At some point in the past, we had the idea of 2 simultaneous FlyingPigeon because the new one is so different we wanted to keep the old one around for a short while. This PR could have helped.

So do not view this PR as Finch 2 anymore, view this PR as a mechanism to quickly deploy another bird, any bird, without having to change code.

This opens up many options, testing is just one of the possible usage.

birdhouse/common.env

@@ -3,6 +3,11 @@
 # Jupyter single-user server image
 export DOCKER_NOTEBOOK_IMAGE="pavics/workflow-tests:200312"
 
+export FINCH_IMAGE="birdhouse/finch:version-0.5.2"
+export GENERIC_BIRD_IMAGE="$FINCH_IMAGE"
+export GENERIC_BIRD_PORT="8010"
+export GENERIC_BIRD_NAME="generic_bird"
+
 # Folder on the host to persist Jupyter user data (noteboooks, HOME settings)
 export JUPYTERHUB_USER_DATA_DIR="/data/jupyterhub_user_data"
 

birdhouse/docker-compose.yml

@@ -151,7 +151,7 @@ services:
     restart: always
 
   finch:
-    image: birdhouse/finch:version-0.5.2
+    image: ${FINCH_IMAGE}
     container_name: finch
     environment:
       HOSTNAME: $HOSTNAME

birdhouse/optional-components/README.md

@@ -51,3 +51,33 @@ server.
 
 No Twitcher/Magpie access control, this Test Thredds is directly behind the
 Nginx proxy.
+
+
+## A generic bird WPS service
+
+Can be used to quickly deploy any birds temporarily without changing code.
+Good to preview new birds or test alternative configuration of existing birds.
+
+How to enable in `env.local` (a copy from
+[`env.local.example`](../env.local.example)):
+
+* Add `./optional-components/generic_bird` to `EXTRA_CONF_DIRS`.
+
+* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`, `GENERIC_BIRD_NAME` in `env.local`
+  for further customizations.  Default values are in [`common.env`](../common.env).
+
+The WPS service will be available at `http://PAVICS_FQDN:GENERIC_BIRD_PORT/wps`
+or `https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/GENERIC_BIRD_NAME` where
+`PAVICS_FQDN`, `PAVICS_FQDN_PUBLIC` and `TWITCHER_PROTECTED_PATH` are defined
+in your `env.local`.
+
+Use same docker image as regular Finch by default but can be customized.
+
+Use a separate Postgres DB for this optional component to be completely
+self-contained and to allow experimenting with different versions of Postgres
+DB.
+
+Magpie will be automatically configured to give complete public anonymous
+access for this WPS service.
+
+Canarie monitoring will also be automatically configured for this WPS service.

birdhouse/optional-components/generic_bird/.gitignore

@@ -0,0 +1,4 @@
+wps.cfg
+generic_bird-magpie-permission.cfg
+generic_bird-magpie-provider.cfg
+generic_bird_canarie_api_monitoring.py

birdhouse/optional-components/generic_bird/docker-compose-extra.yml

@@ -0,0 +1,40 @@
+version: '2.1'
+services:
+  generic_bird:
+    image: ${GENERIC_BIRD_IMAGE}
+    container_name: generic_bird
+    environment:
+      PYWPS_CFG: /wps.cfg
+    ports:
+      - "${GENERIC_BIRD_PORT}:${GENERIC_BIRD_INTERNAL_PORT:-5000}"
+    volumes:
+      - ./optional-components/generic_bird/wps.cfg:/wps.cfg:ro
+      - wps_outputs:/data/wpsoutputs
+      - /tmp
+    links:
+      - postgres_generic_bird
+    restart: always
+
+  postgres_generic_bird:
+    image: ${GENERIC_BIRD_POSTGRES_IMAGE:-postgres:10.12}
+    container_name: postgres_generic_bird
+    volumes:
+      - postgres_generic_bird:/var/lib/postgresql/data/pgdata
+      - ./optional-components/generic_bird/postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+    environment:
+      PGDATA: /var/lib/postgresql/data/pgdata
+    env_file:
+      - ./config/postgres/credentials.env
+    restart: always
+
+  magpie:
+    volumes:
+    - ./optional-components/generic_bird/generic_bird-magpie-permission.cfg:/opt/local/src/magpie/config/permissions/generic_bird-magpie-permission.cfg:ro
+    - ./optional-components/generic_bird/generic_bird-magpie-provider.cfg:/opt/local/src/magpie/config/providers/generic_bird-magpie-provider.cfg:ro
+
+  proxy:
+    volumes:
+    - ./optional-components/generic_bird/generic_bird_canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/generic_bird_canarie_api_monitoring.py:ro
+
+volumes:
+  postgres_generic_bird: {}

birdhouse/optional-components/generic_bird/generic_bird-magpie-permission.cfg.template

@@ -0,0 +1,15 @@
+permissions:
+  - service: ${GENERIC_BIRD_NAME}
+    permission: getcapabilities
+    group: anonymous
+    action: create
+
+  - service: ${GENERIC_BIRD_NAME}
+    permission: describeprocess
+    group: anonymous
+    action: create
+
+  - service: ${GENERIC_BIRD_NAME}
+    permission: execute
+    group: anonymous
+    action: create

birdhouse/optional-components/generic_bird/generic_bird-magpie-provider.cfg.template

@@ -0,0 +1,8 @@
+providers:
+  ${GENERIC_BIRD_NAME}:
+    url: http://${PAVICS_FQDN}:${GENERIC_BIRD_PORT}/wps
+    title: ${GENERIC_BIRD_NAME}
+    public: true
+    c4i: false
+    type: wps
+    sync_type: wps

birdhouse/optional-components/generic_bird/generic_bird_canarie_api_monitoring.py.template

@@ -0,0 +1,14 @@
+SERVICES['node']['monitoring'].update({
+    '${GENERIC_BIRD_NAME}-public': {
+        'request': {
+            'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/${GENERIC_BIRD_NAME}?service=WPS&version=1.0.0&request=GetCapabilities'
+        },
+    },
+    '${GENERIC_BIRD_NAME}': {
+        'request': {
+            'url': 'http://${PAVICS_FQDN}:${GENERIC_BIRD_PORT}/wps?service=WPS&version=1.0.0&request=GetCapabilities'
+        }
+    },
+})
+
+# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python

birdhouse/optional-components/generic_bird/postgres/.gitignore

@@ -0,0 +1 @@
+credentials.env

birdhouse/optional-components/generic_bird/postgres/credentials.env.template

@@ -0,0 +1,3 @@
+POSTGRES_USER=${POSTGRES_PAVICS_USERNAME}
+POSTGRES_PASSWORD=${POSTGRES_PAVICS_PASSWORD}
+POSTGRES_DB=pavics

birdhouse/optional-components/generic_bird/postgres/docker-entrypoint-initdb.d/create-wps-databases.sql

@@ -0,0 +1 @@
+CREATE DATABASE generic_bird;

birdhouse/optional-components/generic_bird/wps.cfg.template

@@ -0,0 +1,9 @@
+[server]
+outputurl = https://${PAVICS_FQDN_PUBLIC}/wpsoutputs
+outputpath = /data/wpsoutputs
+
+[logging]
+level = INFO
+database=postgresql://${POSTGRES_PAVICS_USERNAME}:${POSTGRES_PAVICS_PASSWORD}@postgres_generic_bird/generic_bird
+
+${EXTRA_PYWPS_CONFIG}

birdhouse/pavics-compose.sh

@@ -46,6 +46,8 @@ OPTIONAL_VARS='
   $AUTODEPLOY_PLATFORM_FREQUENCY
   $AUTODEPLOY_NOTEBOOK_FREQUENCY
   $AUTODEPLOY_EXTRA_SCHEDULER_JOBS
+  $GENERIC_BIRD_PORT
+  $GENERIC_BIRD_NAME
 '
 
 # we switch to the real directory of the script, so it still works when used from $PATH

birdhouse/scripts/clear-running-wps-jobs-in-db.sh

@@ -15,8 +15,12 @@ else
     shift
 fi
 
+if [ -z "$POSTGRES_CONTAINER_NAME" ]; then
+    POSTGRES_CONTAINER_NAME=postgres
+fi
+
 set -x
-docker exec postgres psql -U $POSTGRES_USER $DB_NAME -c "select * from pywps_requests where percent_done > -1 and percent_done < 100.0;"
+docker exec $POSTGRES_CONTAINER_NAME psql -U $POSTGRES_USER $DB_NAME -c "select * from pywps_requests where percent_done > -1 and percent_done < 100.0;"
 
 set +x
 echo "
@@ -27,4 +31,4 @@ Clear those jobs? (Ctrl-C to cancel, any keys to continue)"
 read a
 
 set -x
-docker exec postgres psql -U $POSTGRES_USER $DB_NAME -c "delete from pywps_requests where percent_done > -1 and percent_done < 100.0;"
+docker exec $POSTGRES_CONTAINER_NAME psql -U $POSTGRES_USER $DB_NAME -c "delete from pywps_requests where percent_done > -1 and percent_done < 100.0;"