Fix proxy lost log rotation, filling up disk and hogging CPU and RAM to parse large logs on restart

CHANGES.md

@@ -15,7 +15,28 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Fixes
+
+- Proxy lost log rotation, filling up disk and hogging CPU and RAM to parse large logs on restart
+
+  Broken since `2.3.0` (https://github.com/bird-house/birdhouse-deploy/pull/452).
+
+  The root cause is the log rotation has been removed from the python code in
+  https://github.com/Ouranosinc/CanarieAPI/pull/18 without any replacement.
+
+  Fixes https://github.com/bird-house/birdhouse-deploy/issues/593.
+
+  The fix is to restore logrotation in the proxy container using `cron` and
+  `logrotate` instead of the old python code.  Retention and frequency are the
+  same as the previous python code.  For this we needed a custom build of the
+  official Nginx docker image + `cron` + `logrotate`.
+
+  This quick and least disruptive fix to get the production server out of the
+  water should be a temporary solution until a better solution using container
+  STDOUT parsing is implemented for the CanarieAPI and prometheus-log-parser
+  (https://github.com/bird-house/birdhouse-deploy/issues/618).  Then we can
+  switch back to the regular official Nginx image.
+
 
 [2.20.0](https://github.com/bird-house/birdhouse-deploy/tree/2.20.0) (2025-12-10)
 ------------------------------------------------------------------------------------------------------------------

birdhouse/components/proxy/cron.d/logrotate

@@ -0,0 +1,2 @@
+# Match previous CanarieApi https://github.com/Ouranosinc/CanarieAPI/blob/cc0ae59231ee4b58a34571bd12097c660aefb2e3/canarieapi-cron#L1
+* * * * * root /usr/sbin/logrotate -v /etc/logrotate.d/nginx

birdhouse/components/proxy/default.env

@@ -1,6 +1,6 @@
 # All env in this default.env can be overridden by env.local.
 
-export PROXY_IMAGE="nginx:1.23.4"
+export PROXY_IMAGE="pavics/nginx-cron-logrotate:1.23.4-251205"
 
 # Timeout for reading a response from the proxied server.
 # Any WPS processes taking longer than this should use async mode.

birdhouse/components/proxy/docker-compose-extra.yml

@@ -16,6 +16,9 @@ services:
       - ./components/proxy/conf.d:/etc/nginx/conf.d
       - ./components/proxy/nginx.conf:/etc/nginx/nginx.conf
       - ./components/proxy/static:/static
+      - ./components/proxy/nginx-docker-entrypoint.d/launch_cron.sh:/docker-entrypoint.d/launch_cron.sh:ro
+      - ./components/proxy/cron.d:/file/cron.d:ro
+      - ./components/proxy/logrotate.d:/file/logrotate.d:ro
     environment:
       # https://github.com/bird-house/birdhouse-deploy/issues/198
       REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt

birdhouse/components/proxy/docker/Dockerfile

@@ -0,0 +1,6 @@
+# docker buildx build -f Dockerfile . -t pavics/nginx-cron-logrotate:1.23.4-251205 --pull --no-cache
+FROM nginx:1.23.4
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --no-install-suggests -y cron logrotate && \
+    apt-get clean

birdhouse/components/proxy/docker/README.md

@@ -0,0 +1,5 @@
+# pavics/nginx-cron-logrotate
+
+Official Nginx docker image with additional `cron` and `logrotate` installed so
+Nginx can log to a file instead of to STDOUT as with the official Nginx image
+default config.

birdhouse/components/proxy/logrotate.d/nginx

@@ -0,0 +1,17 @@
+# Taken from Nginx docker image, tweaked lightly to match previous CanarieApi.
+
+/var/log/nginx/*.log {
+        missingok
+        # https://github.com/Ouranosinc/CanarieAPI/blob/cc0ae59231ee4b58a34571bd12097c660aefb2e3/canarieapi/logparser.py#L15
+        rotate 150
+        compress
+        delaycompress
+        notifempty
+        create 640 nginx adm
+        sharedscripts
+        postrotate
+                if [ -f /var/run/nginx.pid ]; then
+                        kill -USR1 `cat /var/run/nginx.pid`
+                fi
+        endscript
+}

birdhouse/components/proxy/nginx-docker-entrypoint.d/launch_cron.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -x
+
+# Those config files need to have proper ownership and permissions, else the
+# deamon will not run.
+deploy_config() {
+    srcfile="$1"
+    destfile="$2"
+    cp -v "$srcfile" "$destfile"
+    chown root:root "$destfile"
+    chmod 644 "$destfile"
+}
+
+
+deploy_config /file/cron.d/logrotate /etc/cron.d/logrotate
+deploy_config /file/logrotate.d/nginx /etc/logrotate.d/nginx
+
+cron