Creating docker-compose.prod.yml for use in TrueNAS, etc. Reworking Dockerfiles to enable this new functionality. Leaving docker-compose.yml for use in local development.

Running alembic migrations whenever api container is started.

Adding DEVELOPMENT.md document to help a developer start with WROLPi.

Author: lrnselfreliance

DEVELOPMENT.md

@@ -0,0 +1,22 @@
+This document contains instructions to set up your system for WROLPi development.
+
+# Setup
+
+## Debian 12
+
+1. Install accessory packages necessary for testing:
+    * `sudo apt install ffmpeg chromium chromium-driver sysstat aria2 nodejs npm`
+2. Create Python environment:
+    * `python -m venv venv`
+3. Activate the new environment:
+    * `. venv/bin/activate`
+4. Build the docker containers:
+    * `./scripts/dev_build.sh`
+5. Start the docker containers:
+    * `docker compose up`
+6. Browse to the development containers:
+    * https://localhost:8443
+7. Install npm dependencies used outside the app:
+    * `npm install [email protected] [email protected]`
+8. Run the tests. All tests should pass.
+    * `pytest`

alembic/env.py

@@ -1,4 +1,5 @@
 import os
+import sys
 from logging.config import fileConfig
 
 from alembic import context

docker-compose.yml

@@ -1,5 +1,4 @@
 # Development containers, these should only be used if you are developing.
-version: '3'
 services:
   db: # https://github.com/docker-library/docs/tree/master/postgres
     image: postgres:12
@@ -15,6 +14,7 @@ services:
       - ./pg_data:/var/lib/postgresql/data
 
   api:
+    image: lrnselfreliance/wrolpi:wrolpi-api-latest
     depends_on:
       - db
     build:
@@ -28,7 +28,6 @@ services:
       - './test:/opt/wrolpi/test'
       - './test:/media/wrolpi'
       - './alembic:/opt/wrolpi/alembic'
-      - './alembic.ini:/opt/wrolpi/alembic.ini'
     ports:
       - ${REACT_APP_API-0.0.0.0:8081}:8081
     user: '${UID-1000}:${GID-1000}'
@@ -37,6 +36,7 @@ services:
       interval: 2m
 
   archive: # Python/Sanic wrapper around Singlefile and Readability.
+    image: lrnselfreliance/wrolpi:wrolpi-archive-latest
     build:
       context: .
       dockerfile: docker/archive/Dockerfile
@@ -49,6 +49,7 @@ services:
       interval: 2m
 
   app: # React App
+    image: lrnselfreliance/wrolpi:wrolpi-app-latest
     depends_on:
       - api
     build:
@@ -59,7 +60,7 @@ services:
       - './app/src:/app/src'
       - './app/package.json:/app/package.json'
       - '.env:/app/.env'
-    command: 'npm run start'
+    command: 'serve -s build -l 3000'
     stdin_open: true
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:3000' ]
@@ -84,24 +85,24 @@ services:
       interval: 1m
 
   map_https: # HTTPS for map tile server
+    image: lrnselfreliance/wrolpi:wrolpi-map_https-latest
     depends_on:
       - map
-    environment:
-      - UPSTREAM_HOST=map
     build:
       context: .
       dockerfile: docker/https_proxy/Dockerfile
+      args:
+        - UPSTREAM_HOST=map
     ports:
       - '8084:443'
-    volumes:
-      - './docker/map/nginx.map.conf:/etc/nginx/nginx.conf'
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:443' ]
       interval: 1m
 
   web:
     depends_on:
       - app
+    image: lrnselfreliance/wrolpi:wrolpi-web-latest
     build:
       context: .
       dockerfile: docker/web/Dockerfile
@@ -119,6 +120,7 @@ services:
       interval: 1m
 
   zim: # kiwix-serve
+    image: lrnselfreliance/wrolpi:wrolpi-zim-latest
     build:
       context: .
       dockerfile: docker/zim/Dockerfile
@@ -130,22 +132,22 @@ services:
       interval: 1m
 
   zim_https: # HTTPS for Zim service.
+    image: lrnselfreliance/wrolpi:wrolpi-zim_https-latest
     depends_on:
       - zim
-    environment:
-      - UPSTREAM_HOST=zim
     build:
       context: .
       dockerfile: docker/https_proxy/Dockerfile
+      args:
+        - UPSTREAM_HOST=zim
     ports:
       - '8085:443'
-    volumes:
-      - './docker/zim/nginx.zim.conf:/etc/nginx/nginx.conf'
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:443' ]
       interval: 1m
 
   help: # The WROLPi Help documentation.
+    image: lrnselfreliance/wrolpi:wrolpi-help-latest
     build:
       context: .
       dockerfile: docker/help/Dockerfile
@@ -156,17 +158,17 @@ services:
       interval: 1m
 
   help_https: # HTTPS for Help service.
+    image: lrnselfreliance/wrolpi:wrolpi-help_https-latest
     depends_on:
       - help
-    environment:
-      - UPSTREAM_HOST=help
     build:
       context: .
       dockerfile: docker/https_proxy/Dockerfile
+      args:
+        - UPSTREAM_HOST=help
+        - UPSTREAM_PORT=8086
     ports:
       - '8086:443'
-    volumes:
-      - './docker/help/nginx.help.conf:/etc/nginx/nginx.conf'
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:443' ]
       interval: 1m

docker/api/Dockerfile

@@ -2,20 +2,30 @@ FROM python:3.11-bookworm
 ENV DOCKER=true
 WORKDIR /opt/wrolpi
 
-# Update dependencies for the services install.
-RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping
-RUN ffmpeg -version
-# yt-dlp needs cache directory
+# Update dependencies and install required packages
+RUN apt-get update && \
+    apt-get install -y ffmpeg catdoc sysstat aria2 postgresql-client iputils-ping && \
+    ffmpeg -version && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create cache directory for yt-dlp
 RUN mkdir /.cache && chown -R 1000:1000 /.cache
 
-# Install dependencies.
+# Install Python dependencies
 COPY requirements.txt /opt/wrolpi/requirements.txt
-RUN pip3 install -r /opt/wrolpi/requirements.txt
+RUN pip3 install --no-cache-dir -r /opt/wrolpi/requirements.txt
 
-# Install WROLPi.
+# Copy WROLPi application files
 COPY main.py /opt/wrolpi/
 COPY wrolpi /opt/wrolpi/wrolpi
 COPY modules /opt/wrolpi/modules
+COPY alembic /opt/wrolpi/alembic
+COPY docker/api/alembic.ini /opt/wrolpi/alembic.ini
+
+# Copy and set up entrypoint script
+COPY docker/api/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
 
-ENTRYPOINT [ "python3", "-OO", "/opt/wrolpi/main.py"]
-CMD ["-v", "api", "--host", "0.0.0.0"]
+# Use entrypoint script which will run alembic migrations before starting API.
+ENTRYPOINT ["/entrypoint.sh"]

docker/api/alembic.ini

@@ -0,0 +1,89 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = postgresql://postgres:wrolpi@db:5432/wrolpi
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

docker/api/entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+# Wait for PostgreSQL to be ready
+echo "Waiting for PostgreSQL..."
+until pg_isready -h db -p 5432 -U postgres; do
+  echo "PostgreSQL not ready, waiting..."
+  sleep 2
+done
+
+# Run Alembic migrations
+echo "Running Alembic migrations..."
+python3 -m alembic upgrade head
+
+# Start Sanic service
+echo "Starting Sanic service..."
+exec python3 -OO /opt/wrolpi/main.py -v api --host 0.0.0.0

docker/app/Dockerfile

@@ -1,21 +1,20 @@
-# base image
-FROM node:18-bullseye
-
+FROM node:18-alpine
 WORKDIR /app
-ENV PATH /app/node_modules/.bin:$PATH
+ENV PATH=/app/node_modules/.bin:$PATH
+
+# Install serve and production dependencies
+RUN npm install -g serve && \
+    npm cache clean --force
 
-# install and cache app dependencies
-USER root
-RUN npm install -g serve
-COPY app/public /app/public
-COPY app/src /app/src
-COPY app/package.json app/package-lock.json /app/
-# "node" user will run the app after build.
-RUN chown -R node:node /app
+# Copy package files and install production dependencies
+COPY app/package.json app/package-lock.json ./
+RUN npm ci --omit=dev && \
+    npm cache clean --force
 
-# Install dependencies as "node", this is to avoid changing ownership of many files.
-USER node
-RUN npm install
+# Copy source files and build
+COPY app/public ./public
+COPY app/src ./src
+RUN npm run build
 
-# Run the production build
-CMD ["npm", "run", "start"]
+# Serve the production build
+CMD ["serve", "-s", "build", "-l", "3000"]

docker/archive/Dockerfile

@@ -1,21 +1,29 @@
 FROM browserless/chrome:1.61.1-chrome-stable
+WORKDIR /app
+ENV PYTHONUNBUFFERED=1
 
+# Switch to root for package installation
 USER root
-WORKDIR /app
 
-# Install Python for main.py
-ENV PYTHONUNBUFFERED=1
-RUN apt update && apt install -y python3 python3-dev python3-venv curl && ln -sf python3 /usr/bin/python
+# Install Python and dependencies
+RUN apt-get update && \
+    apt-get install -y python3 python3-dev python3-venv curl && \
+    ln -sf python3 /usr/bin/python && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set up virtual environment
 RUN python3 -m venv /app/venv
-RUN . /app/venv/bin/activate
 
-# Install a newer singlefile and Readability.
+# Install Node.js dependencies
 RUN npm install -g [email protected] [email protected]
 
-# Install the Python wrapper and it's requirements.
+# Install Python dependencies
 COPY docker/archive/requirements.txt /app/requirements.txt
-RUN  /app/venv/bin/pip3 install -r /app/requirements.txt
-COPY docker/archive /app
+RUN /app/venv/bin/pip3 install --no-cache-dir -r /app/requirements.txt
+
+# Copy application files
+COPY docker/archive/main.py /app/
 
-# Launch Archive app in the virtual environment.
-ENTRYPOINT [ "/app/venv/bin/python3", "/app/main.py"]
+# Launch Archive app in virtual environment
+ENTRYPOINT ["/app/venv/bin/python3", "/app/main.py"]