Enable local deployment to improve development and testing

.gitignore

@@ -30,3 +30,4 @@ venv/
 ## Testing
 .pytest_cache/
 *.log
+birdhouse/data/

CHANGES.md

@@ -15,7 +15,49 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Enable local deployment to improve development and testing
+
+  The Birdhouse stack can now be deployed locally and accessed on a browser on the host machine without the need 
+  for an SSL certificate. This is useful for local development and for running tests against the full stack while developing and in CI environments.
+
+  To enable this, add the new `optional-components/local-dev-test` component to `BIRDHOUSE_EXTRA_CONF_DIRS` and
+  set the following environment variables in the local environment file:
+
+  * `export BIRDHOUSE_FQDN=host.docker.internal`
+  * `export BIRDHOUSE_HTTP_ONLY=True`
+
+  You should also add ``host.docker.internal`` to your ``/etc/hosts`` file pointing to the loopback address so 
+  that URLs generated by Birdhouse that refer to ``host.docker.internal`` will resolve properly in a browser:
+
+  ```
+  echo '127.0.0.1    host.docker.internal' | sudo tee -a /etc/hosts
+  ```
+
+  After deploying the stack, you can now interact with the Birdhouse software at ``http://host.docker.internal`` 
+  from the machine that is the docker host.
+
+  In order to implement the changes above, the following non-breaking changes have been made to the deployment code:
+
+  - added a configuration variable `BIRDHOUSE_HTTP_ONLY` which is not set by default. If set to `True` the `proxy` component will only serve content over `http` (not `https`).
+  - added the following configuration variables. These should not be set directly unless you really know what you're doing:
+    - `BIRDHOUSE_PROXY_SCHEME`: default remains `https`. If `BIRDHOUSE_HTTP_ONLY` is `True` then the default becomes `http`
+    - `PROXY_INCLUDE_HTTPS`: default remains `include /etc/nginx/conf.d/https.include;`. If `BIRDHOUSE_HTTP_ONLY` is `True`, the default is that the variable is unset. 
+  - changed the default values for the following configuration variables:
+    - `BIRDHOUSE_ALLOW_UNSECURE_HTTP`: default remains `""`. If `BIRDHOUSE_HTTP_ONLY` is `True` then the default becomes `True`.
+  - logs are written to stderr by default. Previously they were written to stdout.
+    - this allows us to call scripts and programmatically use their outputs. Previously log entries would need to be 
+      manually filtered out before program outputs could be used.
+    - added the `--log-stdout` and `--log-file` flags to the `bin/birdhouse` interface to allow redirecting logs to 
+      stdout or to a specific file instead.
+    - log redirection can also now be set using environment variables:
+      - `BIRDHOUSE_LOG_FD` can be used to redirect logs to a file descriptor (ex: `BIRDHOUSE_LOG_FD=3`)
+      - `BIRDHOUSE_LOG_FILE` can be used to redirect logs to file (ex: `BIRDHOUSE_LOG_FILE=/some/file/on/disk.log`)
+      - Note that the variables here should not be set in the local environment file since that file is sourced **after**
+        some logs are written. Instead, set these by exporting them in the parent process that calls `bin/birdhouse`.
+    - for backwards compatibility, if scripts are not called through the `bin/birdhouse` interface, logs will still be 
+      written to stdout.
 
 [2.6.5](https://github.com/bird-house/birdhouse-deploy/tree/2.6.5) (2024-12-18)
 ------------------------------------------------------------------------------------------------------------------

bin/birdhouse

@@ -2,12 +2,23 @@
 
 THIS_FILE="$(readlink -f "$0" || realpath "$0")"
 THIS_DIR="$(dirname "${THIS_FILE}")"
+THIS_BASENAME="$(basename "${THIS_FILE}")"
 COMPOSE_DIR="$(dirname "${THIS_DIR}")/birdhouse"
 
 export BIRDHOUSE_COMPOSE="${BIRDHOUSE_COMPOSE:-"${COMPOSE_DIR}/birdhouse-compose.sh"}"
 export __BIRDHOUSE_SUPPORTED_INTERFACE=True
 
-USAGE="USAGE: $0 [-h|--help] [-b|--backwards-compatible] [-e|--env-file local-env-file] {info|compose|configs}"
+USAGE="USAGE: $THIS_BASENAME [-h|--help] 
+                             [-b|--backwards-compatible]
+                             [-e|--env-file local-env-file]
+                             [-q|--quiet] [-q|--quiet {DEBUG|INFO|WARN|ERROR|CRITICAL}]
+                             [-s|--log-stdout] [-s|--log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL}]
+                             [-l|--log-file log-file-path]
+                             [-l|--log-file {DEBUG|INFO|WARN|ERROR|CRITICAL} log-file-path]
+                             [-L|--log-level {DEBUG|INFO|WARN|ERROR|CRITICAL}]
+                             {info|compose|configs}"
+USAGE=$(echo $USAGE | tr "\n" " ")
+
 HELP="$USAGE
 
 Manage the Birdhouse software stack.
@@ -18,12 +29,19 @@ Commands:
   configs   Load or execute commands in the Birdhouse configuration environment
 
 Options:
-  -h, --help                   Print this message and exit
-  -b, --backwards-compatible   Run in backwards compatible mode
-  -e, --env-file string        Override the local environment file, default is ${COMPOSE_DIR}/env.local
+  -h, --help                                              Print this message and exit
+  -b, --backwards-compatible                              Run in backwards compatible mode
+  -e, --env-file string                                   Override the local environment file, default is ${COMPOSE_DIR}/env.local
+  -s, --log-stdout                                        Write logs to stdout for all log levels, default is to write to stderr
+  -s, --log-stdout {DEBUG|INFO|WARN|ERROR|CRITICAL}       Write logs to stdout for the given log level only (this option can be repeated)
+  -l, --log-file path                                     Write logs to this file path for all log levels
+  -l, --log-file {DEBUG|INFO|WARN|ERROR|CRITICAL} path    Write logs to this file path for the given log level only (this option can be repeated), this takes precedence over the --log-file option for all log levels
+  -q, --quiet                                             Do not write logs to stdout or stderr for all log levels. Logs will still be written to a file if --log-file is set
+  -q, --quiet {DEBUG|INFO|WARN|ERROR|CRITICAL}            Do not write logs to stdout or stderr for the given log level only (this option can be repeated), Logs will still be written to a file if --log-file is set
+  -L, --log-level {DEBUG|INFO|WARN|ERROR}                 Set log level, default is INFO
 "
 
-CONFIGS_USAGE="USAGE: $0 configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}"
+CONFIGS_USAGE="USAGE: $THIS_BASENAME configs [-h|--help] [-d|--default] {[-p|--print-config-command] | [-c|--command command]}"
 CONFIGS_HELP="$CONFIGS_USAGE
 
 Load or execute commands in the Birdhouse configuration environment.
@@ -32,15 +50,16 @@ Options:
   -d, --default                Only load/print a command for the default configuration settings, not those specified by the local environment file
   -p, --print-config-command   Print a command that can be used to load configuration settings as environment variables
   -c, --command string         Execute the given command after loading configuration settings
-  -q, --quiet                  Suppress stdout when loading configuration settings for the '--command' option.
+Deprecated Options:
+  -q, --quiet                  Suppress stdout when loading configuration settings for the '--command' option. [DEPRECATED: use the --quiet option directly under birdhouse instead]
 
 Example Usage:
 
-  $ ${0} configs -c 'echo \${BIRDHOUSE_FQDN}'
+  $ ${THIS_BASENAME} configs -c 'echo \${BIRDHOUSE_FQDN}'
   example.com  # This is the value of BIRDHOUSE_FQDN as determined by the current configuration settings
-  $ ${0} configs -p
+  $ ${THIS_BASENAME} configs -p
   . /path/to/configs/file/to/source && read_configs
-  $ eval \$(${0} configs)
+  $ eval \$(${THIS_BASENAME} configs)
   $ echo \${BIRDHOUSE_FQDN}
   example.com  # This is the value of BIRDHOUSE_FQDN as determined by the current configuration settings
 "
@@ -148,7 +167,7 @@ parse_configs_args() {
         print_config_command
       elif [ "${CONFIGS_CMD+set}" = 'set' ]; then
         if [ "${CONFIGS_QUIET}" = "True" ]; then
-          eval "$(print_config_command)" > /dev/null
+          eval "$(print_config_command)" 2> /dev/null
         else
           eval "$(print_config_command)"
         fi
@@ -164,6 +183,15 @@ parse_configs_args() {
   esac
 }
 
+# Echos "True" if the first argument is a valid log level
+check_log_dest_override() {
+  case "$1" in 
+    DEBUG|INFO|WARN|ERROR|CRITICAL)
+      echo True
+    ;;
+  esac
+}
+
 # Parse arguments and options
 parse_args() {
   case "$1" in
@@ -194,6 +222,54 @@ parse_args() {
       shift
       parse_args "$@"
       ;;
+    -q|--quiet)
+      shift
+      if [ "$(check_log_dest_override "$1")" ]; then
+        export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:quiet:"
+        shift
+      else
+        export BIRDHOUSE_LOG_QUIET=True # The argument here takes precedence over the env variable
+      fi
+      parse_args "$@"
+      ;;
+    -s|--log-stdout)
+      shift
+      if [ "$(check_log_dest_override "$1")" ]; then
+        export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:fd:1"
+        shift
+      else
+        export BIRDHOUSE_LOG_FD=1 # The argument here takes precedence over the env variable
+      fi
+      parse_args "$@"
+      ;;
+    -l=*|--log-file=*)
+      arg_value="${1#*=}"
+      shift
+      parse_args --log-file "${arg_value}" "$@"
+      ;;
+    -l|--log-file)
+      shift
+      # Note: cannot log to a file named DEBUG, INFO, WARN, ERROR, or CRITICAL
+      if [ "$(check_log_dest_override "$1")" ]; then
+        export BIRDHOUSE_LOG_DEST_OVERRIDE="${BIRDHOUSE_LOG_DEST_OVERRIDE}:$1:file:$(realpath -- "$2")"
+        shift
+      else
+        export BIRDHOUSE_LOG_FILE=$(realpath -- "$1") # The argument here takes precedence over the env variable
+      fi
+      shift
+      parse_args "$@"
+      ;;
+    -L=*|--log-level=*)
+      arg_value="${1#*=}"
+      shift
+      parse_args --log-level "${arg_value}" "$@"
+      ;;
+    -L|--log-level)
+      shift
+      export BIRDHOUSE_LOG_LEVEL="$1" # The argument here takes precedence over the env variable
+      shift
+      parse_args "$@"
+      ;;
     info)
       shift
       "${BIRDHOUSE_COMPOSE}" info "$@"
@@ -212,7 +288,7 @@ parse_args() {
       echo "$HELP"
       ;;
     -??*)
-      parse_multiple_short_flags parse_configs_args "$@"
+      parse_multiple_short_flags parse_args "$@"
       ;;
     *)
       >&2 echo "$USAGE"

birdhouse/README.rst

@@ -291,6 +291,57 @@ Starting and managing the lifecycle of the VM:
    # not needed normally during tight development loop
    vagrant provision
 
+Deploy locally for development or test purposes
+-----------------------------------------------
+
+If you are developing this code base or want to test out a new feature locally on a machine, you may want to deploy 
+the Birdhouse stack locally.
+
+There are two strategies available to deploy the Birdhouse stack locally:
+
+- `Use HTTP scheme deployment`_
+- `Use a Self-Signed SSL certificate`_ 
+
+Use HTTP scheme deployment
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To deploy locally, enable the :ref:`local-dev-test` component. Also set the following two variables in your local
+environment file:
+
+- ``export BIRDHOUSE_FQDN=host.docker.internal``
+- ``export BIRDHOUSE_HTTP_ONLY=True``
+
+This will allow you to access the Birdhouse software in a browser on your local machine using 
+the URL ``http://host.docker.internal`` without the need for an SSL certificate or to expose ports 80 and 443 
+publicly.
+
+Use a Self-Signed SSL certificate
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The `Use HTTP scheme deployment`_ strategy described above will send all information over ``http`` instead of using 
+``https``.
+
+If there are any features that you want to test locally using ``https``, you can deploy locally using a self-signed
+SSL certificate.
+
+You may also need to add the following to the ``docker compose`` settings for the ``twitcher`` component if you're 
+not able to access protected URLs:
+
+.. code:: yaml
+
+  services:
+    twitcher:
+      environment:
+        REQUESTS_CA_BUNDLE: "${BIRDHOUSE_SSL_CERTIFICATE}"
+      volumes:
+        - "${BIRDHOUSE_SSL_CERTIFICATE}:${BIRDHOUSE_SSL_CERTIFICATE}:ro"
+
+
+.. warning::
+
+  Self-signed certificates are not fully supported by the components of the Birdhouse stack and some features may
+  not be fully functional when self-signed certificates are enabled. For example, accessing other components through
+  the JupyterLab interface may fail with an ``SSLError``.
 
 Framework tests
 ---------------

birdhouse/birdhouse-compose.sh

@@ -105,8 +105,8 @@ fi
 
 create_compose_conf_list # this sets COMPOSE_CONF_LIST
 log INFO "Displaying resolved compose configurations:"
-echo "COMPOSE_CONF_LIST="
-echo ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f'
+log INFO "COMPOSE_CONF_LIST="
+log INFO ${COMPOSE_CONF_LIST} | tr ' ' '\n' | grep -v '^-f'
 
 if [ x"$1" = x"info" ]; then
   log INFO "Stopping before execution of docker-compose command."
@@ -123,12 +123,17 @@ if [ x"$1" = x"up" ]; then
       log INFO "Executing '$COMPONENT_PRE_COMPOSE_UP'"
       sh ${SHELL_EXEC_FLAGS} "$COMPONENT_PRE_COMPOSE_UP"
     fi
+    COMPONENT_PRE_COMPOSE_UP_INCLUDE="$adir/pre-docker-compose-up.include"
+    if [ -f "$COMPONENT_PRE_COMPOSE_UP_INCLUDE" ]; then
+      log INFO "Sourcing '$COMPONENT_PRE_COMPOSE_UP_INCLUDE'"
+      . "$COMPONENT_PRE_COMPOSE_UP_INCLUDE"
+    fi
   done
 fi
 
 log INFO "Executing docker-compose with extra options: $* ${COMPOSE_EXTRA_OPTS}"
-# the PROXY_SECURE_PORT is a little trick to make the compose file invalid without the usage of this wrapper script
-PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ${COMPOSE_EXTRA_OPTS}
+# the PROXY_HTTP_PORT is a little trick to make the compose file invalid without the usage of this wrapper script
+PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} $* ${COMPOSE_EXTRA_OPTS}
 ERR=$?
 if [ ${ERR} -gt 0 ]; then
   log ERROR "docker-compose error, exit code ${ERR}"
@@ -148,11 +153,11 @@ while [ $# -gt 0 ]
 do
   if [ x"$1" = x"up" ]; then
     # we restart the proxy after an up to make sure nginx continue to work if any container IP address changes
-    PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} restart proxy
+    PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} restart proxy
 
     # run postgres post-startup setup script
     # Note: this must run before the post-docker-compose-up scripts since some may expect postgres databases to exist
-    postgres_id=$(PROXY_SECURE_PORT=443 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} ps -q postgres 2> /dev/null)
+    postgres_id=$(PROXY_HTTP_PORT=80 HOSTNAME=${BIRDHOUSE_FQDN} docker-compose ${COMPOSE_CONF_LIST} ps -q postgres 2> /dev/null)
     if [ ! -z "$postgres_id" ]; then
       docker exec ${postgres_id} /postgres-setup.sh
     fi