Merge remote-tracking branch 'origin/expert-mode' into expert-mode

Author: Rainbrand

automation/inventory.example

@@ -16,9 +16,9 @@
 
 # if dcs_exists: false and dcs_type: "consul"
 [consul_instances]  # recommendation: 3 or 5-7 nodes
-#10.128.64.140 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1
-#10.128.64.142 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1
-#10.128.64.143 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1
+#10.128.64.140 consul_node_role=server consul_datacenter=dc1
+#10.128.64.142 consul_node_role=server consul_datacenter=dc1
+#10.128.64.143 consul_node_role=server consul_datacenter=dc1
 #10.128.64.144 consul_node_role=client consul_datacenter=dc2
 #10.128.64.145 consul_node_role=client consul_datacenter=dc2
 

automation/molecule/default/converge.yml

@@ -17,7 +17,6 @@
         dcs_type: "{{ 'etcd' if ansible_distribution_major_version in ['10'] or ansible_distribution_release in ['trixie']
           else (['etcd', 'consul'] | random) }}" # TODO: Consul support for RHEL 10, Debian 13
         consul_node_role: server # if dcs_type: "consul"
-        consul_bootstrap_expect: true # if dcs_type: "consul"
         postgresql_version: 18
         pgbouncer_processes: 2 # Test multiple pgbouncer processes (so_reuseport)
         patroni_tags: "datacenter=dc1,key1=value1"

automation/molecule/pg_upgrade/converge.yml

@@ -17,7 +17,6 @@
         dcs_type: "{{ 'etcd' if ansible_distribution_major_version in ['10'] or ansible_distribution_release in ['trixie']
           else (['etcd', 'consul'] | random) }}" # TODO: Consul support for RHEL 10, Debian 13
         consul_node_role: server # if dcs_type: "consul"
-        consul_bootstrap_expect: true # if dcs_type: "consul"
         postgresql_version: 17 # redefine the version to install for the upgrade test
         pgbouncer_processes: 4 # Test multiple pgbouncer processes (so_reuseport)
         cacheable: true

automation/playbooks/consul_cluster.yml

@@ -148,14 +148,6 @@
         consul_dnsmasq_servers: "{{ consul_dnsmasq_servers | reject('equalto', '127.0.0.1') | list }}"
       when: dcs_type | default('etcd') == "consul" and consul_dnsmasq_enable | default(true) | bool and ('127.0.0.1' in (consul_dnsmasq_servers | default([])))
 
-    # Setting variables for Consul during cloud deployment
-    - name: Redefine the consul_node_role and consul_bootstrap_expect variables
-      ansible.builtin.set_fact:
-        consul_node_role: "{{ 'server' if not dcs_exists | default(false) else 'client' }}"
-        consul_bootstrap_expect: "{{ not dcs_exists | default(false) }}"
-        consul_datacenter: "{{ server_location | default('dc1') }}"
-      when: cloud_provider | default('') | length > 0
-
   roles:
     - role: vitabaks.autobase.firewall
       vars:

automation/requirements.txt

@@ -1,5 +1,5 @@
-ansible==12.1.0
-boto3==1.40.61
+ansible==12.2.0
+boto3==1.40.74
 dopy==0.3.7
-google-auth==2.42.0
-hcloud==2.9.0
+google-auth==2.43.0
+hcloud==2.11.1

automation/requirements.yml

@@ -1,24 +1,24 @@
 ---
 collections:
   - name: amazon.aws
-    version: ">=10.1.1"
+    version: ">=10.1.2"
   - name: community.aws
     version: ">=10.0.0"
   - name: google.cloud
-    version: ">=1.8.0"
+    version: ">=1.10.2"
   - name: azure.azcollection
-    version: ">=3.8.0"
+    version: ">=3.10.1"
   - name: community.digitalocean
     version: ">=1.27.0"
   - name: hetzner.hcloud
-    version: ">=5.2.0"
+    version: ">=5.4.0"
   - name: community.postgresql
-    version: ">=3.14.2"
+    version: ">=4.1.0"
   - name: community.docker
-    version: ">=4.6.1"
+    version: ">=4.8.2"
   - name: community.general
-    version: ">=10.7.2"
+    version: ">=11.4.1"
   - name: ansible.posix
-    version: ">=1.6.2"
+    version: ">=2.1.0"
   - name: ansible.utils
-    version: ">=5.1.2"
+    version: ">=6.0.0"

automation/roles/cloud_resources/defaults/main.yml

@@ -62,14 +62,14 @@ azure_blob_storage_absent: false # Allow to delete Azure Blob Storage when delet
 
 digital_ocean_spaces_create: true # if 'cloud_provider=digitalocean'
 digital_ocean_spaces_name: "{{ patroni_cluster_name }}-backup" # Name of the Spaces Object Storage (S3 bucket).
-digital_ocean_spaces_region: "nyc3" # The region to create the Space in.
+digital_ocean_spaces_region: "{{ (server_location in ['nyc1', 'nyc2']) | ternary('nyc3', server_location) }}" # The region to create the Space in.
 digital_ocean_spaces_access_key: "" # (required) Spaces Object Storage ACCESS KEY
 digital_ocean_spaces_secret_key: "" # (required) Spaces Object Storage SECRET KEY
 digital_ocean_spaces_absent: false # Allow to delete Spaces Object Storage when deleting a cluster servers using the 'state=absent' variable.
 
 hetzner_object_storage_create: true # if 'cloud_provider=hetzner'
 hetzner_object_storage_name: "{{ patroni_cluster_name }}-backup" # Name of the Object Storage (S3 bucket).
-hetzner_object_storage_region: "{{ server_location }}" # The region where the Object Storage (S3 bucket) will be created.
+hetzner_object_storage_region: "{{ (server_location in ['hel1', 'fsn1', 'nbg1']) | ternary(server_location, 'nbg1') }}" # The region where the Object Storage (S3 bucket) will be created.
 hetzner_object_storage_endpoint: "https://{{ hetzner_object_storage_region }}.your-objectstorage.com"
 hetzner_object_storage_access_key: "" # (required) Object Storage ACCESS KEY
 hetzner_object_storage_secret_key: "" # (required) Object Storage SECRET KEY

automation/roles/cloud_resources/tasks/digitalocean.yml

@@ -143,15 +143,15 @@
       ansible.builtin.set_fact:
         default_ip_range: >-
           {{
-            vpc_info.data
+            vpc_info.get('data', [])
             | selectattr('region', 'equalto', server_location)
             | selectattr('default', 'equalto', true)
             | map(attribute='ip_range')
             | first
           }}
       when:
         - server_network | length < 1
-        - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length > 0
+        - vpc_info.get('data', []) | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length > 0
 
     # if server_network is not specified and there is no default VPC, create a network
     - name: "DigitalOcean: Create a VPC '{{ digital_ocean_vpc_name | default('network-' + server_location | default('')) }}'"
@@ -163,32 +163,41 @@
       register: digital_ocean_vpc
       when:
         - server_network | length < 1
-        - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length == 0
+        - vpc_info.get('data', []) | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length == 0
 
     - name: "Set variable: server_network"
       ansible.builtin.set_fact:
         server_network: "{{ digital_ocean_vpc_name | default('network-' + server_location) }}"
-      when: digital_ocean_vpc is changed
+      when:
+        - digital_ocean_vpc is defined
+        - digital_ocean_vpc is changed
 
     - name: "DigitalOcean: Gather information about VPC"
       community.digitalocean.digital_ocean_vpc_info:
         oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}"
       register: vpc_info
-      when: digital_ocean_vpc is changed
+      when:
+        - digital_ocean_vpc is defined
+        - digital_ocean_vpc is changed
 
     # if server_network is specified
     - name: "Fail if no VPC found in the specified region"
       ansible.builtin.fail:
         msg: "No VPC found with name '{{ server_network }}' in region '{{ server_location }}'"
       when:
         - server_network | length > 0
-        - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('name', 'equalto', server_network) | list | length == 0
+        - (vpc_info.get('data', [])
+            | selectattr('region', 'equalto', server_location)
+            | selectattr('name', 'equalto', server_network)
+            | list
+            | length
+          ) == 0
 
     - name: Extract ip_range from VPC "{{ server_network | default('') }}"
       ansible.builtin.set_fact:
         vpc_ip_range: >-
           {{
-            vpc_info.data
+            vpc_info.get('data', [])
             | selectattr('region', 'equalto', server_location)
             | selectattr('name', 'equalto', server_network)
             | map(attribute='ip_range')
@@ -200,7 +209,7 @@
       ansible.builtin.set_fact:
         vpc_id: >-
           {{
-            vpc_info.data
+            vpc_info.get('data', [])
             | selectattr('region', 'equalto', server_location)
             | selectattr('name', 'equalto', server_network)
             | map(attribute='id')
@@ -620,9 +629,12 @@
         oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}"
         name: "{{ digital_ocean_spaces_name }}"
         region: "{{ digital_ocean_spaces_region }}"
-        aws_access_key_id: "{{ AWS_ACCESS_KEY_ID }}"
-        aws_secret_access_key: "{{ AWS_SECRET_ACCESS_KEY }}"
+        aws_access_key_id: "{{ digital_ocean_spaces_access_key | default(AWS_ACCESS_KEY_ID | default(default_access_key), true) }}"
+        aws_secret_access_key: "{{ digital_ocean_spaces_secret_key | default(AWS_SECRET_ACCESS_KEY | default(default_secret_key), true) }}"
         state: present
+      vars:
+        default_access_key: "{{ pgbackrest_s3_key | default(wal_g_aws_access_key_id | default('')) }}"
+        default_secret_key: "{{ pgbackrest_s3_key_secret | default(wal_g_aws_secret_access_key | default('')) }}"
       when:
         - (pgbackrest_install | bool or wal_g_install | bool)
         - digital_ocean_spaces_create | bool

automation/roles/cloud_resources/tasks/hetzner.yml

@@ -448,20 +448,21 @@
       amazon.aws.s3_bucket:
         endpoint_url: "{{ hetzner_object_storage_endpoint }}"
         ceph: true
-        aws_access_key: "{{ hetzner_object_storage_access_key }}"
-        aws_secret_key: "{{ hetzner_object_storage_secret_key }}"
+        aws_access_key: "{{ hetzner_object_storage_access_key | default(default_access_key, true) }}"
+        aws_secret_key: "{{ hetzner_object_storage_secret_key | default(default_secret_key, true) }}"
         name: "{{ hetzner_object_storage_name }}"
         region: "{{ hetzner_object_storage_region }}"
         requester_pays: false
         state: present
       register: s3_bucket_result
       failed_when: s3_bucket_result.failed and not "GetBucketRequestPayment" in s3_bucket_result.msg
       # TODO: https://github.com/ansible-collections/amazon.aws/issues/2447
+      vars:
+        default_access_key: "{{ pgbackrest_s3_key | default(wal_g_aws_access_key_id | default('')) }}"
+        default_secret_key: "{{ pgbackrest_s3_key_secret | default(wal_g_aws_secret_access_key | default('')) }}"
       when:
         - (pgbackrest_install | bool or wal_g_install | bool)
         - hetzner_object_storage_create | bool
-        - hetzner_object_storage_access_key | length > 0
-        - hetzner_object_storage_secret_key | length > 0
 
     # Server and volume
     - name: "Hetzner Cloud: Gather information about servers"

automation/roles/cloud_resources/tasks/inventory.yml

@@ -78,6 +78,7 @@
   ansible.builtin.add_host:
     name: "{{ item.private_ip }}"
     group: consul_instances
+    consul_node_role: "{{ 'server' if not dcs_exists | default(false) else 'client' }}"
     ansible_ssh_host: "{{ item[server_public_ip | bool | ternary('public_ip', 'private_ip')] }}"
     ansible_ssh_private_key_file: "{{ ssh_private_key_file | default(None) }}"
     new_node: "{{ item.new_node | default(omit) }}"