remove proxy dependency on log rotation

Author: mishaschwartz

CHANGES.md

@@ -15,7 +15,45 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Remove `proxy` component's dependency on `scheduler` and `scheduler-job-logrotate-nginx`
+
+  Creates new settings in `optional-components/proxy-log-volume` that create the `proxy-logs` docker volume as well
+  as instructing Nginx to write access logs to an additional log file (specified by `PROXY_LOG_PATH`). These settings
+  are included as a `COMPONENT_DEPENDENCY` in components that require access to the the `proxy` access logs as a
+  regular file. If no components require access to these logs as a regular file then the `proxy` component will only
+  write access logs to the stdout stream for that container.
+
+  Right now, the only components that require access to logs in this way are `components/canarie-api` and
+  `optional-components/prometheus-log-parser`. Both of these now include `optional-components/proxy-log-volume` as a
+  `COMPONENT_DEPENDENCY`.
+
+  Note: this means that if no optional components require `optional-components/proxy-log-volume` as a dependency
+  then logs from the `proxy` container will only be written to stdout/stderr. This means that there is no need
+  for any additional custom log rotation handling since the logs are handled directly by docker. This means that
+  the `proxy` service itself no longer need to be dependant on the `scheduler` and `scheduler-job-logrotate-nginx`
+  components.
+
+  Note: a previous discussion suggested that logs could be parsed directly from the stdout stream of the `proxy`
+  container. However, there is no way to do so that doesn't require very hacky workarounds. Possible solutions that
+  were explored and rejected include:
+
+    - Mounting the log file from the `proxy` container from the host to the relevant containers.
+      Rejected because this practice is highly discouraged by docker as the actual storage location of log files
+      is not standardized and may be changed in future versions.
+    - Writing logs to a named pipe or socket within the `proxy` container.
+      Rejected because this is very difficult to set up and is untested when then mounted to other containers. 
+      Also, a different named pipe would be required for each consumer which is currently very difficult to set up 
+      using birdhouse's deployment tools. 
+
+  **Breaking Change**: if a custom component (not included in this repository) uses the `proxy-logs` named volume.
+  It must now include `optional-components/proxy-log-volume` as a `COMPONENT_DEPENDENCY` for that custom component.
+
+  **Breaking Change**: if `SCHEDULER_JOB_BACKUP_ARGS` specifies `-l proxy` explicitly (not `-l '*'`) then this should
+  be changed to `-l proxy-log-volume` since the backup script has been moved. Note that it is not necessary to
+  specify `-l proxy-log-volume` if `--birdhouse-logs` is also specified because the log data is identical in both.
+
 
 [2.20.2](https://github.com/bird-house/birdhouse-deploy/tree/2.20.2) (2026-01-05)
 ------------------------------------------------------------------------------------------------------------------

birdhouse/components/canarie-api/config/proxy/docker-compose-extra.yml

@@ -2,4 +2,3 @@ services:
   proxy:
     volumes:
       - ./components/canarie-api/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/canarie-api:ro
-      - proxy-logs:${PROXY_LOG_DIR}

birdhouse/components/canarie-api/default.env

@@ -13,4 +13,5 @@ export DELAYED_EVAL="
 # add any component that this component requires to run
 COMPONENT_DEPENDENCIES="
   ./components/proxy
+  ./optional-components/proxy-log-volume
 "

birdhouse/components/canarie-api/docker-compose-extra.yml

@@ -46,6 +46,6 @@ services:
     entrypoint: /entrypoint
     logging: *default-logging
 
+# Note: proxy-logs volume is defined in the proxy-logs-volume component
 volumes:
   canarie-data:
-  proxy-logs:

birdhouse/components/proxy/default.env

@@ -35,13 +35,9 @@ export PROXY_INCLUDE_FOR_PORT_80='$([ x"$BIRDHOUSE_ALLOW_UNSECURE_HTTP" = x"True
 export PROXY_LISTEN_443_PARAMS=""
 export PROXY_LISTEN_80_PARAMS=""
 
-export PROXY_LOG_DIR="/var/log/nginx/"
-export PROXY_LOG_FILE="access_file.log"
-export PROXY_LOG_PATH='${PROXY_LOG_DIR}/${PROXY_LOG_FILE}'
 
 export DELAYED_EVAL="
   $DELAYED_EVAL
-  PROXY_LOG_PATH
   BIRDHOUSE_PROXY_SCHEME
   BIRDHOUSE_ALLOW_UNSECURE_HTTP
   PROXY_INCLUDE_HTTPS
@@ -65,15 +61,7 @@ export OPTIONAL_VARS="
   \$PROXY_INCLUDE_FOR_PORT_80
   \$PROXY_READ_TIMEOUT_VALUE
   \$BIRDHOUSE_PROXY_ROOT_LOCATION
-  \$PROXY_LOG_FILE
-  \$PROXY_LOG_PATH
   \$PROXY_INCLUDE_HTTPS
   \$PROXY_LISTEN_443_PARAMS
   \$PROXY_LISTEN_80_PARAMS
 "
-
-# TODO: remove this dependency once https://github.com/bird-house/birdhouse-deploy/issues/618 is resolved
-COMPONENT_DEPENDENCIES="
-  ./components/scheduler
-  ./optional-components/scheduler-job-logrotate-nginx
-"

birdhouse/components/proxy/nginx.conf.template

@@ -16,13 +16,11 @@ http {
     default_type  application/octet-stream;
 
     # NOTE: if this log_format ever changes, make sure to update the relevant code in the
-    #       prometheus-log-parser component as well to match the change.
+    #       components that assume this log format.
     log_format  main  '$remote_addr - $remote_user [$time_iso8601] "$request" '
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for"';
 
-    access_log  ${PROXY_LOG_PATH}  main;
-
     sendfile        on;
     #tcp_nopush     on;
 

birdhouse/optional-components/README.rst

@@ -692,3 +692,41 @@ Note that you do *not* need an SSL certificate set up to deploy the stack in thi
 .. warning:: 
 
   **DO NOT** enable this component in production. This is intended for local development and test purposes only!
+
+Proxy Log Volume
+----------------
+
+This optional setting creates a named docker volume `proxy-logs` that contains the logs directory for the `proxy` component.
+
+It also creates an Nginx configuration that instructs the proxy service to write access logs to a regular file in that directory.
+
+.. note::
+
+    By default, access logs are only written to the stdout stream of the `proxy` docker container.
+
+.. note::
+
+    Because access logs are now being written to a regular file, enabling this component will also enable the 
+    `optional-components/scheduler-job-logrotate-nginx` scheduler job to ensure that this file is rotated and that it will not
+    get too big.
+
+.. warning::
+
+    **DO NOT** enable this setting directly. It will be enabled as a component dependency by other components that require access
+    to the `proxy` access logs as a regular file.
+
+
+If you are creating a custom component that requires access to the `proxy` access logs, add the following to that component's
+`default.env` file:
+
+.. code::shell
+
+    COMPONENT_DEPENDENCIES="
+        ./optional-components/proxy-log-volume
+    "
+
+This will ensure that the proxy log volume setting will be enabled. You can then mount the volume named `proxy-logs` to any container
+that your custom component creates and read the `proxy` access logs at a file defined by the configuration variable `PROXY_LOG_FILE`.
+
+For example, if `PROXY_LOG_FILE` is set to ``access_file.log`` (the default) and you mount the `proxy-logs` volume to the ``/logs``
+directory in your container, the `proxy` access logs can be read at ``/logs/access_file.log`` in your container.

birdhouse/optional-components/prometheus-log-parser/config/proxy/docker-compose-extra.yml

@@ -1,12 +1,10 @@
 services:
   proxy:
     volumes:
+      # Note: proxy-logs volume is defined in the proxy-logs-volume component
       - proxy-logs:${PROXY_LOG_DIR}
   prometheus-log-parser:
     volumes:
       - proxy-logs:/var/log/proxy
     environment:
       - PROXY_LOG_FILE=${PROXY_LOG_FILE}
-
-volumes:
-  proxy-logs:

birdhouse/optional-components/prometheus-log-parser/default.env

@@ -16,3 +16,8 @@ export DELAYED_EVAL="
   $DELAYED_EVAL
   PROMETHEUS_LOG_PARSER_IMAGE
 "
+
+COMPONENT_DEPENDENCIES="
+  ./components/proxy
+  ./optional-components/proxy-log-volume
+"

birdhouse/optional-components/proxy-log-volume/.gitignore

@@ -0,0 +1 @@
+config/proxy/conf.extra-directives.d/proxy-log.conf