Remove proxy component's dependency on scheduler and scheduler-job-logrotate-nginx

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "2.20.4"
+current_version = "2.21.0"
 commit = true
 tag = false
 tag_name = "{new_version}"

CHANGES.md

@@ -15,8 +15,50 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
+[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+
+[2.21.0](https://github.com/bird-house/birdhouse-deploy/tree/2.21.0) (2026-01-27)
+------------------------------------------------------------------------------------------------------------------
+
 ## Changes
 
+- Remove `proxy` component's dependency on `scheduler` and `scheduler-job-logrotate-nginx`
+
+  Creates new settings in `optional-components/proxy-log-volume` that create the `proxy-logs` docker volume as well
+  as instructing Nginx to write access logs to an additional log file (specified by `PROXY_LOG_PATH`). These settings
+  are included as a `COMPONENT_DEPENDENCY` in components that require access to the the `proxy` access logs as a
+  regular file. If no components require access to these logs as a regular file then the `proxy` component will only
+  write access logs to the stdout stream for that container.
+
+  Right now, the only components that require access to logs in this way are `components/canarie-api` and
+  `optional-components/prometheus-log-parser`. Both of these now include `optional-components/proxy-log-volume` as a
+  `COMPONENT_DEPENDENCY`.
+
+  Note: this means that if no optional components require `optional-components/proxy-log-volume` as a dependency
+  then logs from the `proxy` container will only be written to stdout/stderr. This means that there is no need
+  for any additional custom log rotation handling since the logs are handled directly by docker. This means that
+  the `proxy` service itself no longer need to be dependant on the `scheduler` and `scheduler-job-logrotate-nginx`
+  components.
+
+  Note: a previous discussion suggested that logs could be parsed directly from the stdout stream of the `proxy`
+  container. However, there is no way to do so that doesn't require very hacky workarounds. Possible solutions that
+  were explored and rejected include:
+
+    - Mounting the log file from the `proxy` container from the host to the relevant containers.
+      Rejected because this practice is highly discouraged by docker as the actual storage location of log files
+      is not standardized and may be changed in future versions.
+    - Writing logs to a named pipe or socket within the `proxy` container.
+      Rejected because this is very difficult to set up and is untested when then mounted to other containers. 
+      Also, a different named pipe would be required for each consumer which is currently very difficult to set up 
+      using birdhouse's deployment tools. 
+
+  **Breaking Change**: if a custom component (not included in this repository) uses the `proxy-logs` named volume.
+  It must now include `optional-components/proxy-log-volume` as a `COMPONENT_DEPENDENCY` for that custom component.
+
+  **Breaking Change**: if `SCHEDULER_JOB_BACKUP_ARGS` specifies `-l proxy` explicitly (not `-l '*'`) then this should
+  be changed to `-l proxy-log-volume` since the backup script has been moved. Note that it is not necessary to
+  specify `-l proxy-log-volume` if `--birdhouse-logs` is also specified because the log data is identical in both.
+
 - Make docker compose logging options configurable
 
   Introduce a new component `components/logging` that sets default logging options for all docker 

Makefile

@@ -8,7 +8,7 @@ override BIRDHOUSE_MAKE_DIR := $(shell realpath -P $$(dirname $(BIRDHOUSE_MAKE_C
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 2.20.4
+override APP_VERSION := 2.21.0
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -18,13 +18,13 @@ for a full-fledged production platform.
     * - citation
       - | |citation|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.20.4.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.21.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.20.4...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.21.0...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-2.20.4-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-2.21.0-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.20.4
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.21.0
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)

RELEASE.txt

@@ -1 +1 @@
-2.20.4 2026-01-16T16:40:32Z
+2.21.0 2026-01-27T20:49:01Z

birdhouse/components/canarie-api/config/proxy/docker-compose-extra.yml

@@ -2,4 +2,3 @@ services:
   proxy:
     volumes:
       - ./components/canarie-api/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/canarie-api:ro
-      - proxy-logs:${PROXY_LOG_DIR}

birdhouse/components/canarie-api/default.env

@@ -12,5 +12,5 @@ export DELAYED_EVAL="
 
 # add any component that this component requires to run
 COMPONENT_DEPENDENCIES="
-  ./components/proxy
+  ./optional-components/proxy-log-volume
 "

birdhouse/components/canarie-api/docker-compose-extra.yml

@@ -36,6 +36,6 @@ services:
     restart: always
     entrypoint: /entrypoint
 
+# Note: proxy-logs volume is defined in the proxy-logs-volume component
 volumes:
   canarie-data:
-  proxy-logs:

birdhouse/components/canarie-api/docker_configuration.py.template

@@ -108,8 +108,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.20.4',
-            'releaseTime': '2026-01-16T16:40:32Z',
+            'version': '2.21.0',
+            'releaseTime': '2026-01-27T20:49:01Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',
@@ -141,8 +141,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.20.4',
-            'releaseTime': '2026-01-16T16:40:32Z',
+            'version': '2.21.0',
+            'releaseTime': '2026-01-27T20:49:01Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',

birdhouse/components/proxy/default.env

@@ -35,13 +35,9 @@ export PROXY_INCLUDE_FOR_PORT_80='$([ x"$BIRDHOUSE_ALLOW_UNSECURE_HTTP" = x"True
 export PROXY_LISTEN_443_PARAMS=""
 export PROXY_LISTEN_80_PARAMS=""
 
-export PROXY_LOG_DIR="/var/log/nginx/"
-export PROXY_LOG_FILE="access_file.log"
-export PROXY_LOG_PATH='${PROXY_LOG_DIR}/${PROXY_LOG_FILE}'
 
 export DELAYED_EVAL="
   $DELAYED_EVAL
-  PROXY_LOG_PATH
   BIRDHOUSE_PROXY_SCHEME
   BIRDHOUSE_ALLOW_UNSECURE_HTTP
   PROXY_INCLUDE_HTTPS
@@ -65,15 +61,7 @@ export OPTIONAL_VARS="
   \$PROXY_INCLUDE_FOR_PORT_80
   \$PROXY_READ_TIMEOUT_VALUE
   \$BIRDHOUSE_PROXY_ROOT_LOCATION
-  \$PROXY_LOG_FILE
-  \$PROXY_LOG_PATH
   \$PROXY_INCLUDE_HTTPS
   \$PROXY_LISTEN_443_PARAMS
   \$PROXY_LISTEN_80_PARAMS
 "
-
-# TODO: remove this dependency once https://github.com/bird-house/birdhouse-deploy/issues/618 is resolved
-COMPONENT_DEPENDENCIES="
-  ./components/scheduler
-  ./optional-components/scheduler-job-logrotate-nginx
-"

birdhouse/components/proxy/nginx.conf.template

@@ -16,13 +16,11 @@ http {
     default_type  application/octet-stream;
 
     # NOTE: if this log_format ever changes, make sure to update the relevant code in the
-    #       prometheus-log-parser component as well to match the change.
+    #       components that assume this log format.
     log_format  main  '$remote_addr - $remote_user [$time_iso8601] "$request" '
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for"';
 
-    access_log  ${PROXY_LOG_PATH}  main;
-
     sendfile        on;
     #tcp_nopush     on;
 

birdhouse/optional-components/README.rst

@@ -692,3 +692,41 @@ Note that you do *not* need an SSL certificate set up to deploy the stack in thi
 .. warning:: 
 
   **DO NOT** enable this component in production. This is intended for local development and test purposes only!
+
+Proxy Log Volume
+----------------
+
+This optional setting creates a named docker volume `proxy-logs` that contains the logs directory for the `proxy` component.
+
+It also creates an Nginx configuration that instructs the proxy service to write access logs to a regular file in that directory.
+
+.. note::
+
+    By default, access logs are only written to the stdout stream of the `proxy` docker container.
+
+.. note::
+
+    Because access logs are now being written to a regular file, enabling this component will also enable the 
+    `optional-components/scheduler-job-logrotate-nginx` scheduler job to ensure that this file is rotated and that it will not
+    get too big.
+
+.. warning::
+
+    **DO NOT** enable this setting directly. It will be enabled as a component dependency by other components that require access
+    to the `proxy` access logs as a regular file.
+
+
+If you are creating a custom component that requires access to the `proxy` access logs, add the following to that component's
+`default.env` file:
+
+.. code::shell
+
+    COMPONENT_DEPENDENCIES="
+        ./optional-components/proxy-log-volume
+    "
+
+This will ensure that the proxy log volume setting will be enabled. You can then mount the volume named `proxy-logs` to any container
+that your custom component creates and read the `proxy` access logs at a file defined by the configuration variable `PROXY_LOG_FILE`.
+
+For example, if `PROXY_LOG_FILE` is set to ``access_file.log`` (the default) and you mount the `proxy-logs` volume to the ``/logs``
+directory in your container, the `proxy` access logs can be read at ``/logs/access_file.log`` in your container.

birdhouse/optional-components/prometheus-log-parser/config/proxy/docker-compose-extra.yml

@@ -1,12 +1,10 @@
 services:
   proxy:
     volumes:
+      # Note: proxy-logs volume is defined in the proxy-logs-volume component
       - proxy-logs:${PROXY_LOG_DIR}
   prometheus-log-parser:
     volumes:
       - proxy-logs:/var/log/proxy
     environment:
       - PROXY_LOG_FILE=${PROXY_LOG_FILE}
-
-volumes:
-  proxy-logs:

birdhouse/optional-components/prometheus-log-parser/default.env

@@ -16,3 +16,7 @@ export DELAYED_EVAL="
   $DELAYED_EVAL
   PROMETHEUS_LOG_PARSER_IMAGE
 "
+
+COMPONENT_DEPENDENCIES="
+  ./optional-components/proxy-log-volume
+"

birdhouse/optional-components/proxy-log-volume/.gitignore

@@ -0,0 +1 @@
+config/proxy/conf.extra-directives.d/proxy-log.conf

birdhouse/optional-components/proxy-log-volume/config/proxy/conf.extra-directives.d/proxy-log.conf.template

@@ -0,0 +1,2 @@
+    access_log  ${PROXY_LOG_PATH}  main;
+    access_log  /var/log/nginx/access.log  main;  # this is a symlink to /dev/stdout in the proxy container

birdhouse/optional-components/proxy-log-volume/config/proxy/docker-compose-extra.yml

@@ -0,0 +1,8 @@
+services:
+  proxy:
+    volumes:
+      - ./optional-components/proxy-log-volume/config/proxy/conf.extra-directives.d:/etc/nginx/conf.extra-directives.d/proxy-log-volume:ro
+      - proxy-logs:${PROXY_LOG_DIR}
+
+volumes:
+  proxy-logs:

birdhouse/optional-components/proxy-log-volume/default.env

@@ -0,0 +1,24 @@
+# PROXY_LOG_DIR is set to a different directory than the other nginx logs (/var/log/nginx) because the other
+# nginx logs are written to files that are actually symlinks to the stdout and stderr streams. We do not want
+# to share these symlinks in a volume because they will necessarily not point to the correct target once they are
+# mounted in a different container. To avoid potential confusion if trying to read from these symlinked log files
+# in a different container, it is better to just not make them shareable (as a volume) in the first place.
+export PROXY_LOG_DIR="/logs/"
+export PROXY_LOG_FILE="access_file.log"
+export PROXY_LOG_PATH='${PROXY_LOG_DIR}/${PROXY_LOG_FILE}'
+
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  PROXY_LOG_PATH
+"
+
+export OPTIONAL_VARS="
+  $OPTIONAL_VARS
+  \$PROXY_LOG_FILE
+  \$PROXY_LOG_PATH
+"
+
+COMPONENT_DEPENDENCIES="
+  ./components/scheduler
+  ./optional-components/scheduler-job-logrotate-nginx
+"

birdhouse/optional-components/proxy-log-volume/pre-docker-compose-up.include

@@ -0,0 +1,3 @@
+if ! echo "${BIRDHOUSE_EXTRA_CONF_DIRS}" | grep -q 'scheduler-job-logrotate-nginx[[:space:]]*$'; then
+  log WARN 'Access logs for the proxy component are being written to a regular file but no log rotation is enabled. You may want to enable the scheduler and scheduler-job-logrotate-nginx components.'
+fi

birdhouse/optional-components/scheduler-job-logrotate-nginx/config.yml.template

@@ -1,12 +1,13 @@
 - name: logrotate-nginx
   comment: Rotate log files for nginx
   schedule: '${SCHEDULER_JOB_LOGROTATE_NGINX_JOB_SCHEDULE}'
-  command: bash -c 'cp /etc/logrotate.conf.orig /etc/logrotate.conf && chown root:root /etc/logrotate.conf && chmod 644 /etc/logrotate.conf && /usr/sbin/logrotate -v /etc/logrotate.conf'
+  command: bash -c 'cp /etc/logrotate.conf.orig /etc/logrotate.conf && chown root:root /etc/logrotate.conf && chmod 644 /etc/logrotate.conf && /usr/sbin/logrotate -v /etc/logrotate.conf >> /logs/logrotate-nginx.log 2>&1'
   dockerargs: >-
     --rm --name logrotate-nginx
     --volume ${COMPOSE_PROJECT_NAME}_proxy-logs:/var/log/nginx/:rw
     --volume ${COMPOSE_DIR}/optional-components/scheduler-job-logrotate-nginx/nginx.logrotate:/etc/logrotate.conf.orig:ro
+    --volume ${BIRDHOUSE_LOG_DIR}:/logs:rw
   trigger:
     - command: sh -c '[ -f /var/run/nginx.pid ] && kill -USR1 $(cat /var/run/nginx.pid)'
       container: proxy
-  image: 'stakater/logrotate:3.13.0'
+  image: '${SCHEDULER_JOB_LOGROTATE_NGINX_IMAGE}'

birdhouse/optional-components/scheduler-job-logrotate-nginx/default.env

@@ -1,13 +1,20 @@
-export SCHEDULER_JOB_LOGROTATE_NGINX_JOB_SCHEDULE="* * * * *"  # UTC
-export SCHEDULER_JOB_LOGROTATE_NGINX_ROTATE=150
-export SCHEDULER_JOB_LOGROTATE_NGINX_SIZE=1M
+export SCHEDULER_JOB_LOGROTATE_NGINX_JOB_SCHEDULE="*/10 * * * *"  # UTC
+export SCHEDULER_JOB_LOGROTATE_NGINX_ROTATE=50
+export SCHEDULER_JOB_LOGROTATE_NGINX_SIZE=20M
+
+export SCHEDULER_JOB_LOGROTATE_NGINX_DOCKER='stakater/logrotate'
+export SCHEDULER_JOB_LOGROTATE_NGINX_VERSION='3.13.0'
+export SCHEDULER_JOB_LOGROTATE_NGINX_IMAGE='${SCHEDULER_JOB_LOGROTATE_NGINX_DOCKER}:${SCHEDULER_JOB_LOGROTATE_NGINX_VERSION}'
 
 VARS="
   $VARS
   \$SCHEDULER_JOB_LOGROTATE_NGINX_JOB_SCHEDULE
   \$SCHEDULER_JOB_LOGROTATE_NGINX_ROTATE
   \$SCHEDULER_JOB_LOGROTATE_NGINX_SIZE
+  \$SCHEDULER_JOB_LOGROTATE_NGINX_IMAGE
 "
 
-# TODO: deprecate this component once https://github.com/bird-house/birdhouse-deploy/issues/618 is resolved
-#       remember that deprecating involves adding an entry to optional-components/.gitignore
+export DELAYED_EVAL="
+  $DELAYED_EVAL
+  SCHEDULER_JOB_LOGROTATE_NGINX_IMAGE
+"

birdhouse/optional-components/scheduler-job-logrotate-nginx/pre-docker-compose-up.include

@@ -0,0 +1,3 @@
+if ! echo "${ALL_CONF_DIRS}" | grep -q 'proxy-log-volume[[:space:]]*$'; then
+  log WARN 'The scheduler-job-logrotate-nginx scheduler job is enabled but proxy access logs are not being written to a regular file that needs rotation. This WILL cause problems! Please disable the scheduler-job-logrotate-nginx job.'
+fi

docs/source/conf.py

@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.20.4'
+version = '2.21.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.20.4'
+release = '2.21.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

tests/unit/test_read_configs_include.py

@@ -197,10 +197,6 @@ class TestReadConfigs(TestReadBasicConfigs):
     ]
 
     default_all_conf_order_with_dependencies: list[str] = [
-        # TODO: remove the scheduler dependencies once 
-        #       https://github.com/bird-house/birdhouse-deploy/issues/618 is resolved
-        "./components/scheduler",
-        "./optional-components/scheduler-job-logrotate-nginx",
         "./components/proxy",
         "./components/magpie",
         "./components/twitcher",