WIP: Add support for deploying in containers

.config/make/docker.mak

@@ -5,8 +5,8 @@ DOCKER_REGISTRY ?= autobase
 # Sanitize the tag by replacing slashes with hyphens for Docker compatibility
 SANITIZED_TAG := $(subst /,-,$(TAG))
 
-.PHONY: docker-lint docker-lint-console-ui docker-lint-console-api docker-lint-console-db docker-lint-console
-docker-lint: docker-lint-automation docker-lint-console-ui docker-lint-console-api docker-lint-console-db docker-lint-console ## Lint all Dockerfiles
+.PHONY: docker-lint docker-lint-console-ui docker-lint-console-api docker-lint-console-db docker-lint-console docker-lint-patroni
+docker-lint: docker-lint-automation docker-lint-console-ui docker-lint-console-api docker-lint-console-db docker-lint-console docker-lint-patroni ## Lint all Dockerfiles
 
 docker-lint-automation: ## Lint automation Dockerfile
 	@echo "Lint automation container Dockerfile"
@@ -33,8 +33,13 @@ docker-lint-console: ## Lint console Dockerfile (all services)
 	docker run --rm -i -v $(PWD)/console/Dockerfile:/Dockerfile \
 	hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3059 --ignore DL4001 /Dockerfile
 
-.PHONY: docker-build docker-build-console-ui docker-build-console-api docker-build-console-db docker-build-console
-docker-build: docker-build-automation docker-build-console-ui docker-build-console-api docker-build-console-db docker-build-console ## Build for all Docker images
+docker-lint-patroni: ## Lint patroni Dockerfile
+	@echo "Lint Patroni container Dockerfile"
+	docker run --rm -i -v $(PWD)/docker/patroni/Dockerfile:/Dockerfile \
+	hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3059 --ignore DL4001 /Dockerfile
+
+.PHONY: docker-build docker-build-console-ui docker-build-console-api docker-build-console-db docker-build-console docker-build-patroni
+docker-build: docker-build-automation docker-build-console-ui docker-build-console-api docker-build-console-db docker-build-console docker-build-patroni ## Build for all Docker images
 
 docker-build-automation: ## Build automation image
 	@echo "Build automation docker image with tag $(TAG) (sanitized as $(SANITIZED_TAG))";
@@ -56,6 +61,10 @@ docker-build-console: ## Build console image (all services)
 	@echo "Build console docker image with tag $(TAG) (sanitized as $(SANITIZED_TAG))"
 	docker build --no-cache --platform linux/amd64 --tag console:$(SANITIZED_TAG) --file console/Dockerfile .
 
+docker-build-patroni: ## Build patroni image
+	@echo "Build Patroni docker image with tag $(TAG) (sanitized as $(SANITIZED_TAG))"
+	docker build --no-cache --platform linux/amd64 --tag patroni:$(SANITIZED_TAG) --file docker/patroni/Dockerfile .
+
 .PHONY: docker-push docker-push-console-ui docker-push-console-api docker-push-console-db docker-push-console
 docker-push: docker-push-automation docker-push-console-ui docker-push-console-api docker-push-console-db docker-push-console ## Push all images to Dockerhub (example: make docker-push TAG=my_tag DOCKER_REGISTRY=my_repo DOCKER_REGISTRY_USER="my_username" DOCKER_REGISTRY_PASSWORD="my_password")
 
@@ -89,6 +98,12 @@ docker-push-console: ## Push console image to Dockerhub (all services)
 	docker tag console:$(SANITIZED_TAG) $(DOCKER_REGISTRY)/console:$(SANITIZED_TAG)
 	docker push $(DOCKER_REGISTRY)/console:$(SANITIZED_TAG)
 
+docker-push-patroni: ## Push patroni image to Dockerhub
+	@echo "Push Patroni docker image with tag $(TAG) (sanitized as $(SANITIZED_TAG))"
+	echo "$(DOCKER_REGISTRY_PASSWORD)" | docker login --username "$(DOCKER_REGISTRY_USER)" --password-stdin
+	docker tag patroni:$(SANITIZED_TAG) $(DOCKER_REGISTRY)/patroni:$(SANITIZED_TAG)
+	docker push $(DOCKER_REGISTRY)/patroni:$(SANITIZED_TAG)
+
 .PHONY: docker-tests
 docker-tests: ## Run tests for docker
 	$(MAKE) docker-lint

README.md

@@ -101,8 +101,6 @@ It requires the installation of a consul in client mode on each application serv
 
 ## Compatibility
 
-RedHat and Debian based distros (x86_64)
-
 ###### Supported Linux Distributions:
 
 - **Debian**: 11, 12, 13
@@ -112,6 +110,8 @@ RedHat and Debian based distros (x86_64)
 - **Rocky Linux**: 8, 9, 10
 - **AlmaLinux**: 8, 9, 10
 
+Architecture: x86_64/amd64, arm64/aarch64
+
 ###### PostgreSQL versions:
 
 all supported PostgreSQL versions

automation/Dockerfile

@@ -1,5 +1,5 @@
 FROM debian:bookworm-slim
-LABEL maintainer="Vitaliy Kukharik [email protected]"
+LABEL maintainer="Autobase <[email protected]>"
 
 USER root
 

automation/playbooks/add_balancer.yml

@@ -123,6 +123,12 @@
     - role: vitabaks.autobase.hostname
     - role: vitabaks.autobase.resolv_conf
 
+    - role: vitabaks.autobase.docker
+      when:
+        - installation_method | default('packages') == "docker"
+        - docker_install | default(true) | bool
+      tags: docker, docker_install
+
     - role: vitabaks.autobase.haproxy
       when: with_haproxy_load_balancing | default(false) | bool
 

automation/playbooks/add_pgnode.yml

@@ -166,6 +166,12 @@
     - role: vitabaks.autobase.copy
     - role: vitabaks.autobase.cron
 
+    - role: vitabaks.autobase.docker
+      when:
+        - installation_method | default('packages') == "docker"
+        - docker_install | default(true) | bool
+      tags: docker, docker_install
+
 - name: vitabaks.autobase.add_pgnode | Configure pgBackRest
   hosts: pgbackrest:postgres_cluster
   become: true
@@ -205,7 +211,7 @@
       when: pg_probackup_install | default(false) | bool
 
     - role: vitabaks.autobase.pgbouncer
-      when: pgbouncer_install | default(false) | bool
+      when: pgbouncer_install | default(true) | bool
 
     - role: vitabaks.autobase.pgpass
 

automation/playbooks/deploy_pgcluster.yml

@@ -108,6 +108,36 @@
         - ansible_os_family == "Debian"
         - "'gnupg' not in ansible_facts.packages or 'apt-transport-https' not in ansible_facts.packages"
 
+    # Ansible requires the iproute package for network facts to be populated
+    - name: Make sure that the iproute is installed
+      ansible.builtin.package:
+        name: iproute
+        state: present
+      register: package_status
+      until: package_status is success
+      delay: 5
+      retries: 3
+      when: ansible_os_family == "RedHat"
+
+    - name: Make sure that the iproute is installed
+      ansible.builtin.apt:
+        name: iproute2
+        state: present
+      register: apt_status
+      until: apt_status is success
+      delay: 5
+      retries: 3
+      when: ansible_os_family == "Debian"
+
+    - name: Make sure that the postgres user exists (for Docker mode)
+      ansible.builtin.user:
+        name: postgres
+        shell: /bin/bash
+        state: present
+      when:
+        - installation_method | default('packages') == "docker"
+        - inventory_hostname in groups['postgres_cluster']
+
     # (optional) Command or script to be executed before the Postgres cluster deployment.
     - block:
         - name: Print pre-deploy command
@@ -186,6 +216,12 @@
         tls_cert_regenerate: "{{ patroni_tls_cert_regenerate | default(false) }}" # Do not generate new certificates if they already exist.
       when: tls_cert_generate | default(true) | bool
 
+    - role: vitabaks.autobase.docker
+      when:
+        - installation_method | default('packages') == "docker"
+        - docker_install | default(true) | bool
+      tags: docker, docker_install
+
 - name: vitabaks.autobase.deploy_pgcluster | Deploy etcd cluster
   ansible.builtin.import_playbook: etcd_cluster.yml
   when: not dcs_exists | default(false) | bool and dcs_type | default('etcd') == "etcd"
@@ -312,7 +348,7 @@
     - role: vitabaks.autobase.cron
 
     - role: vitabaks.autobase.pgbouncer
-      when: pgbouncer_install | default(false) | bool
+      when: pgbouncer_install | default(true) | bool
 
     - role: vitabaks.autobase.pgpass
 

automation/roles/common/defaults/main.yml

@@ -228,6 +228,7 @@ postgresql_data_dir: "\
   {% else %}\
   {{ postgresql_home_dir }}/{{ postgresql_version }}/{{ postgresql_cluster_name }}\
   {% endif %}"
+postgresql_base_dir: "{{ postgresql_data_dir | regex_replace('/$', '') | dirname | dirname }}"
 # You can specify custom WAL dir path. Example: "/pgwal/{{ postgresql_version }}/pg_wal"
 postgresql_wal_dir: "" # if defined, symlink will be created [optional]
 postgresql_conf_dir: "\
@@ -1139,10 +1140,10 @@ confd_package_repo: "https://github.com/kelseyhightower/confd/releases/download/
 packages_from_file: []
 #  - "<package_file_name>.{{ pkg_type }}"
 
-# ----------------------------------------------------------------------------------------------------
+# ----------------------------------------------------------
 # Redefine the installation method (optional)
-# ----------------------------------------------------------------------------------------------------
-installation_method: "packages" # "packages" = install via deb/rpm packages; "docker" = use container images (TODO).
+# ----------------------------------------------------------
+installation_method: "packages" # "packages" = install via deb/rpm packages; "docker" = use container images.
 
 # The Patroni package will be installed from the deb/rpm package by default.
 # You also have the option of choosing an installation method using the pip package.
@@ -1166,6 +1167,100 @@ patroni_pip_requirements_repo: []
 #  - "https://<my_repo>/<package_n>.tar.gz"
 pip_package_repo: "https://bootstrap.pypa.io/get-pip.py" # latest version pip3 for python3 (or use "pip-<version>.tar.gz").
 
+############################################################
+# Docker containers (if installation_method: "docker")
+############################################################
+
+docker_install: true # Set to false if you prefer to install Docker manually.
+docker_add_repo: true # Set to false if you prefer to manage the Docker repository yourself.
+
+postgresql_container:
+  name: "postgres"
+  image: "autobase/patroni:{{ postgresql_container.tag }}" # image with Patroni, PostgreSQL, extensions, and backup tools.
+  tag: "postgresql-{{ postgresql_version }}" # major or minor version, e.g. 17 or 17.5
+  options:
+    - "--network host"
+    - "--shm-size=4g"
+  envs:
+    - "PGDATA={{ postgresql_data_dir }}"
+  volumes:
+    - "/etc/patroni:/etc/patroni"
+    - "{{ patroni_log_dir }}:{{ patroni_log_dir }}"
+    - "/etc/postgresql:/etc/postgresql"
+    - "{{ postgresql_base_dir }}:{{ postgresql_base_dir }}"
+    - "{{ postgresql_log_dir }}:{{ postgresql_log_dir }}"
+    - "{{ postgresql_unix_socket_dir }}:{{ postgresql_unix_socket_dir }}"
+    - "{{ tls_dir }}:{{ tls_dir }}"
+  run_command: |
+    bash -c '/usr/bin/patroni --version && \
+      mkdir -p {{ postgresql_conf_dir }} {{ postgresql_data_dir }}; \
+      chown -R postgres:postgres \
+        {{ postgresql_conf_dir }} \
+        {{ postgresql_data_dir }} \
+        {{ postgresql_log_dir }} \
+        {{ postgresql_unix_socket_dir }}; \
+      exec su - postgres -c "/usr/bin/patroni /etc/patroni/patroni.yml"'
+
+# if dcs_type: "etcd" and dcs_exists: false
+etcd_container:
+  name: "etcd"
+  image: "bitnami/etcd:{{ etcd_container.tag }}"
+  tag: "{{ etcd_version }}"
+  options:
+    - "--network host"
+    - "--env-file {{ etcd_conf_dir }}/etcd.conf"
+  envs: []
+  volumes:
+    - "{{ etcd_data_dir }}:{{ etcd_data_dir }}"
+    - "{{ etcd_tls_dir }}:{{ etcd_tls_dir }}"
+  run_command: ""
+
+# if dcs_type: "consul" and dcs_exists: false
+consul_container:
+  name: "consul"
+  image: "bitnami/consul:{{ consul_container.tag }}"
+  tag: "{{ consul_version }}"
+  options:
+    - "--network host"
+  envs: []
+  volumes:
+    - "{{ consul_config_path }}:{{ consul_config_path }}"
+    - "{{ consul_configd_path }}:{{ consul_configd_path }}"
+    - "{{ consul_data_path }}:{{ consul_data_path }}"
+    - "{{ consul_run_path }}:{{ consul_run_path }}"
+    - "{{ consul_tls_dir }}:{{ consul_tls_dir }}"
+  run_command: |
+    consul agent \
+      -config-file={{ consul_config_path }}/config.json \
+      -config-dir={{ consul_configd_path }} \
+      -pid-file={{ consul_run_path }}/consul.pid
+
+# if pgbouncer_install: true
+pgbouncer_container:
+  name: "pgbouncer"
+  image: "bitnami/pgbouncer:{{ pgbouncer_container.tag }}"
+  tag: "{{ pgbouncer_version | default('latest') }}"
+  options:
+    - "--network host"
+  envs: []
+  volumes:
+    - "{{ pgbouncer_conf_dir }}:{{ pgbouncer_conf_dir }}"
+    - "{{ pgbouncer_log_dir }}:{{ pgbouncer_log_dir }}"
+    - "{{ pgbouncer_tls_dir }}:{{ pgbouncer_tls_dir }}"
+
+# if with_haproxy_load_balancing: true
+haproxy_container:
+  name: "haproxy"
+  image: "bitnami/haproxy:{{ haproxy_container.tag }}"
+  tag: "{{ haproxy_version | default('latest') }}"
+  options:
+    - "--network host"
+  envs: []
+  volumes:
+    - "/etc/haproxy:/etc/haproxy"
+    - "/run/haproxy:/run/haproxy"
+  run_command: "haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /run/haproxy/haproxy.pid"
+
 ############################################################
 # Other variables
 ############################################################

automation/roles/consul/templates/consul_systemd-docker.service.j2

@@ -0,0 +1,37 @@
+[Unit]
+Description=Consul docker wrapper
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=simple
+TimeoutSec=30
+RestartSec=15
+Restart=always
+
+# Delete consul container (if any)
+ExecStartPre=-/usr/bin/docker rm -f {{ consul_container.name | default('consul') }}
+
+# Start consul container
+ExecStart=/usr/bin/docker run -d \
+  --name {{ consul_container.name | default('consul') }} \
+  {% for option in consul_container.options | default([]) %}
+  {{ option }} \
+  {% endfor %}
+  {% for env in consul_container.envs | default([]) %}
+  --env {{ env }} \
+  {% endfor %}
+  {% for volume in consul_container.volumes | default([]) %}
+  --volume {{ volume }} \
+  {% endfor %}
+  {{ consul_container.image | default('bitnami/consul:latest') }} {{ consul_container.run_command | default('') }}
+
+# Stop consul container
+ExecStop=-/usr/bin/docker stop {{ consul_container.name | default('consul') }}
+ExecStopPost=-/usr/bin/docker rm -f {{ consul_container.name | default('consul') }}
+
+# Send HUP to reload config
+ExecReload=/usr/bin/docker exec {{ consul_container.name | default('consul') }} kill -SIGHUP 1
+
+[Install]
+WantedBy=multi-user.target

automation/roles/docker/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Jeff Geerling
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.