Add cluster scaling support for cloud providers (#1267)

Author: vitabaks

automation/playbooks/add_node.yml

@@ -1,5 +1,29 @@
 ---
 - name: vitabaks.autobase.add_node | PostgreSQL HA Cluster Scaling (add a new node)
+  hosts: localhost
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI)
+    # Note: Applicable only for "aws", "gcp", "azure", because:
+    # "digitalocean" - requires the Spaces access keys ("AWS_ACCESS_KEY_ID" and "AWS_SECRET_ACCESS_KEY" variables)
+    # "hetzner" - requires the S3 credentials ("hetzner_object_storage_access_key" and "hetzner_object_storage_secret_key" variables).
+    - name: "Set variable: 'pgbackrest_install' to configure Postgres backups"
+      ansible.builtin.set_fact:
+        pgbackrest_install: true
+      when:
+        - not (pgbackrest_install | default(false) | bool or wal_g_install | default(false) | bool)
+        - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure']
+        - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings
+      tags: always
+  roles:
+    - role: vitabaks.autobase.cloud_resources
+      when: cloud_provider | default('') | length > 0
+      vars:
+        cluster_scaling: true
+      tags: always
+
+- name: vitabaks.autobase.add_node | Prepare and perform pre-checks
   hosts: postgres_cluster:etcd_cluster:consul_instances:balancers
   become: true
   become_method: sudo
@@ -265,22 +289,21 @@
     - role: vitabaks.autobase.pgbackrest
       when: pgbackrest_install | bool
 
-- name: vitabaks.autobase.add_node | Add new PostgreSQL replica to the cluster
-  hosts: new_replica
+- name: vitabaks.autobase.add_node | Regenerate Postgres TLS certificates
+  hosts: postgres_cluster
   become: true
   become_method: sudo
   gather_facts: true
   any_errors_fatal: true
   vars:
     cluster_scaling: true
-
-  pre_tasks:
+  tasks:
     - name: Generate Postgres TLS certificate
       ansible.builtin.include_role:
         name: vitabaks.autobase.tls_certificate
       vars:
         tls_group_name: "postgres_cluster"
-        tls_cert_regenerate: true  # Regenerate when adding nodes
+        tls_cert_regenerate: true
       when: tls_cert_generate | default(true) | bool
 
     - name: Copy Postgres TLS certificate, key and CA to all nodes
@@ -289,6 +312,14 @@
         tasks_from: copy
       when: tls_cert_generate | default(true) | bool
 
+- name: vitabaks.autobase.add_node | Add new PostgreSQL replica to the cluster
+  hosts: new_replica
+  become: true
+  become_method: sudo
+  gather_facts: true
+  any_errors_fatal: true
+  vars:
+    cluster_scaling: true
   roles:
     - role: vitabaks.autobase.wal_g
       when: wal_g_install | default(false) | bool

automation/requirements.yml

@@ -1,17 +1,17 @@
 ---
 collections:
   - name: amazon.aws
-    version: ">=10.1.0"
+    version: ">=10.1.1"
   - name: community.aws
     version: ">=10.0.0"
   - name: google.cloud
-    version: ">=1.7.0"
+    version: ">=1.8.0"
   - name: azure.azcollection
-    version: ">=3.7.0"
+    version: ">=3.8.0"
   - name: community.digitalocean
     version: ">=1.27.0"
   - name: hetzner.hcloud
-    version: ">=5.1.0"
+    version: ">=5.2.0"
   - name: community.postgresql
     version: ">=3.14.2"
   - name: community.docker

automation/roles/cloud_resources/tasks/aws.yml

@@ -240,42 +240,59 @@
         ip_address_type: "{{ 'public_ip_address' if server_public_ip | bool else 'private_ip_address' }}"
 
     # Server and volume
-    - name: "AWS: Create or modify EC2 instance"
-      amazon.aws.ec2_instance:
-        access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}"
-        secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}"
-        name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
-        state: present
-        instance_type: "{{ server_type }}"
-        image_id: "{{ server_image }}"
-        key_name: "{{ ssh_key_name }}"
-        region: "{{ server_location }}"
-        network_interfaces:
-          - subnet_id: "{{ server_network }}"
-            groups: "{{ ([] if not cloud_firewall | bool else [ec2_security_group_result.group_id]) }}"
-            assign_public_ip: "{{ server_public_ip | bool }}"
-            delete_on_termination: true
-        volumes:
-          - device_name: /dev/sda1
-            ebs:
-              volume_type: "{{ system_volume_type | default('gp3', true) }}"
-              volume_size: "{{ system_volume_size | default(80) | int }}"
-              delete_on_termination: true
-          - device_name: /dev/sdb
-            ebs:
-              volume_type: "{{ volume_type | default('gp3', true) }}"
-              volume_size: "{{ volume_size | int }}"
-              delete_on_termination: true
-      loop: "{{ range(0, server_count | int) | list }}"
-      loop_control:
-        index_var: idx
-        label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
-      register: server_result
-      until:
-        - server_result.instances[0][ip_address_type] is defined
-        - server_result.instances[0][ip_address_type] | length > 0
-      retries: 3
-      delay: 10
+    - block:
+        - name: "AWS: Gather information about EC2 instances"
+          amazon.aws.ec2_instance_info:
+            access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}"
+            secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}"
+            region: "{{ server_location }}"
+            filters:
+              instance-type: "{{ server_type }}"
+              instance-state-name: ["pending", "running", "shutting-down", "stopping", "stopped"]
+              "tag:Name": "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
+          loop: "{{ range(0, server_count | int) | list }}"
+          loop_control:
+            index_var: idx
+            label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
+          register: ec2_instance_info
+
+        - name: "AWS: Create EC2 instance"
+          amazon.aws.ec2_instance:
+            access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}"
+            secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}"
+            name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
+            state: running
+            instance_type: "{{ server_type }}"
+            image_id: "{{ server_image }}"
+            key_name: "{{ ssh_key_name }}"
+            region: "{{ server_location }}"
+            network_interfaces:
+              - subnet_id: "{{ server_network }}"
+                groups: "{{ ([] if not cloud_firewall | bool else [ec2_security_group_result.group_id]) }}"
+                assign_public_ip: "{{ server_public_ip | bool }}"
+                delete_on_termination: true
+            volumes:
+              - device_name: /dev/sda1
+                ebs:
+                  volume_type: "{{ system_volume_type | default('gp3', true) }}"
+                  volume_size: "{{ system_volume_size | default(80) | int }}"
+                  delete_on_termination: true
+              - device_name: /dev/sdb
+                ebs:
+                  volume_type: "{{ volume_type | default('gp3', true) }}"
+                  volume_size: "{{ volume_size | int }}"
+                  delete_on_termination: true
+            tags:
+              Name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
+              Cluster: "{{ patroni_cluster_name }}"
+          loop: "{{ range(0, server_count | int) | list }}"
+          loop_control:
+            index_var: idx
+            label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
+          register: ec2_instance_result
+          retries: 3
+          delay: 10
+          when: ec2_instance_info.results[idx].instances | length == 0 # only if instance does not exist
       when: not server_spot | default(aws_ec2_spot_instance | default(false)) | bool
 
     # Spot instance (if 'server_spot' is 'true')
@@ -288,7 +305,6 @@
             filters:
               instance-lifecycle: "spot"
               instance-type: "{{ server_type }}"
-              image-id: "{{ server_image }}"
               instance-state-name: ["pending", "running", "shutting-down", "stopping", "stopped"]
               "tag:Name": "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
           loop: "{{ range(0, server_count | int) | list }}"
@@ -327,6 +343,7 @@
                     delete_on_termination: true
             tags:
               Name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
+              Cluster: "{{ patroni_cluster_name }}"
           loop: "{{ ec2_spot_instance_info.results }}"
           loop_control:
             index_var: idx
@@ -353,20 +370,25 @@
           retries: 3
           delay: 10
           when: item.spot_request.spot_instance_request_id is defined
-
-        # if spot instances are created now
-        - name: "Set variable: server_result"
-          ansible.builtin.set_fact:
-            server_result: "{{ ec2_spot_instance_result }}"
-          when: ec2_spot_instance_result.changed | default(false)
-
-        # if spot instances have already been created
-        - name: "Set variable: server_result"
-          ansible.builtin.set_fact:
-            server_result: "{{ ec2_spot_instance_info }}"
-          when: not ec2_spot_instance_result.changed | default(false)
       when: server_spot | default(aws_ec2_spot_instance | default(false)) | bool
 
+    # Combine existing and newly created instances
+    - name: "Set variable: server_result"
+      ansible.builtin.set_fact:
+        server_result:
+          results: >-
+            {{
+              (
+                (ec2_instance_info.results | default([]))
+                + (ec2_instance_result.results | default([]))
+                + (ec2_spot_instance_info.results | default([]))
+                + (ec2_spot_instance_result.results | default([]))
+              )
+              | selectattr('instances', 'defined')
+              | selectattr('instances', '!=', [])
+              | list
+            }}
+
     # Classic Load Balancer (CLB) - previous generation
     - name: "AWS: Create Classic Load Balancer (CLB)"
       amazon.aws.elb_classic_lb:
@@ -402,13 +424,23 @@
       loop_control:
         label: "{{ patroni_cluster_name }}-{{ item }}"
       register: aws_elb_classic_lb
-      when: cloud_load_balancer | bool and aws_load_balancer_type | lower == 'clb' and
+      when: server_result.results | default([]) | length > 0 and
+        cloud_load_balancer | bool and aws_load_balancer_type | lower == 'clb' and
         (item == 'primary' or
         (item == 'replica' and server_count | int > 1) or
         (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool))
 
     # Network Load Balancer (NLB)
-    - name: "AWS: Create NLB Target Group"
+    - name: Build NLB targets
+      ansible.builtin.set_fact:
+        nlb_targets: "{{ (nlb_targets | default([])) + [{'Id': item, 'Port': target_port | int}] }}"
+      loop: "{{ instance_ids | default([]) }}"
+      vars:
+        instance_ids: "{{ (server_result | default({'results': []})) | json_query('results[].instances[].instance_id') }}"
+        target_port: "{{ pgbouncer_listen_port | default('6432') if pgbouncer_install | bool else postgresql_port | default('5432') }}"
+      when: cloud_load_balancer | bool and aws_load_balancer_type | lower == 'nlb'
+
+    - name: "AWS: Create or modify NLB Target Group"
       community.aws.elb_target_group:
         access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}"
         secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}"
@@ -427,16 +459,7 @@
         healthy_threshold_count: 3
         successful_response_codes: "200"
         target_type: instance
-        targets: >-
-          {{
-            server_result.results
-            | map(attribute='instances')
-            | map('first')
-            | map(attribute='instance_id')
-            | map('community.general.dict_kv', 'Id')
-            | map('combine', {'Port': target_port | int})
-            | list
-          }}
+        targets: "{{ nlb_targets | default([]) }}"
         modify_targets: true
         wait: false
         state: present
@@ -514,7 +537,7 @@
       volume_size: "{{ volume_size }} GB"
       public_ip: "{{ item.instances[0].public_ip_address | default('N/A', true) }}"
       private_ip: "{{ item.instances[0].private_ip_address | default('N/A', true) }}"
-  loop: "{{ server_result.results }}"
+  loop: "{{ server_result.results | default([]) }}"
   loop_control:
     index_var: idx
     label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
@@ -529,7 +552,7 @@
     port: 22
     delay: 5
     timeout: 300
-  loop: "{{ server_result.results }}"
+  loop: "{{ server_result.results | default([]) }}"
   loop_control:
     index_var: idx
     label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
@@ -545,9 +568,20 @@
           {{ ip_addresses | default([]) +
             [{
               'public_ip': item.instances[0].public_ip_address | default(''),
-              'private_ip': item.instances[0].private_ip_address | default('')
+              'private_ip': item.instances[0].private_ip_address | default(''),
+              'new_node': (cluster_scaling | default(false) | bool and item.instances[0].instance_id not in existing_instance_ids)
             }]
           }}
+      vars:
+        existing_instance_ids: >-
+          {{
+            (
+              (ec2_instance_info.results | default([])) +
+              (ec2_spot_instance_info.results | default([]))
+            )
+            | selectattr('instances','defined') | map(attribute='instances') | list | flatten
+            | selectattr('instance_id','defined') | map(attribute='instance_id') | list
+          }}
       loop: "{{ server_result.results | selectattr('instances', 'defined') }}"
       loop_control:
         label: >-