feat(expertMode): implemented expert mode

.config/make/docker.mak

@@ -11,7 +11,7 @@ docker-lint: docker-lint-automation docker-lint-console-ui docker-lint-console-a
 docker-lint-automation: ## Lint automation Dockerfile
 	@echo "Lint automation container Dockerfile"
 	docker run --rm -i -v $(PWD)/automation/Dockerfile:/Dockerfile \
-	hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3059 /Dockerfile
+	hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3013 --ignore DL3059 /Dockerfile
 
 docker-lint-console-ui: ## Lint console ui Dockerfile
 	@echo "Lint console ui container Dockerfile"

automation/Dockerfile

@@ -13,18 +13,21 @@ COPY automation /autobase/automation
 RUN apt-get clean && rm -rf /var/lib/apt/lists/partial \
   && apt-get update -o Acquire::CompressionTypes::Order::=gz \
   && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-  ca-certificates gnupg keychain debian-archive-keyring apt-transport-https \
-  git python3 python3-dev python3-pip ssh-client sshpass gcc g++ cmake make libssl-dev curl lsb-release \
+     ca-certificates gnupg keychain debian-archive-keyring apt-transport-https \
+     git python3 python3-dev python3-pip ssh-client sshpass gcc g++ cmake make libssl-dev curl lsb-release \
+  # fresh pip/setuptools/wheel (fewer builds from source)
+  && python3 -m pip install --break-system-packages --no-cache-dir --upgrade pip setuptools wheel \
   # repo and key for Azure CLI
   && curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.asc.gpg > /dev/null \
   && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/azure-cli.list \
   && apt-get update \
   # requirements
-  && pip3 install --break-system-packages --no-cache-dir -r /autobase/automation/requirements.txt \
+  && pip3 install --break-system-packages --no-cache-dir --retries 3 --timeout 60 \
+    -r /autobase/automation/requirements.txt \
   && ansible-galaxy install --force -r /autobase/automation/requirements.yml \
   && ansible-galaxy collection list \
-  && pip3 install --break-system-packages --no-cache-dir -r \
-  /root/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt \
+  && pip3 install --break-system-packages --no-cache-dir --retries 3 --timeout 60 \
+    -r /root/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt \
   # azure-cli
   && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y azure-cli \
   # cleanup
@@ -34,8 +37,8 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/partial \
   && chmod +x /autobase/automation/entrypoint.sh
 
 # Link collection source for Ansible runtime
-RUN mkdir -p /root/.ansible/collections/ansible_collections/vitabaks && \
-  ln -sfn /autobase/automation /root/.ansible/collections/ansible_collections/vitabaks/autobase
+RUN mkdir -p /root/.ansible/collections/ansible_collections/vitabaks \
+  && ln -sfn /autobase/automation /root/.ansible/collections/ansible_collections/vitabaks/autobase
 
 # Set environment variables
 ENV ANSIBLE_COLLECTIONS_PATH=/root/.ansible/collections/ansible_collections:/usr/local/lib/python3.11/dist-packages/ansible_collections

automation/inventory.example

@@ -16,9 +16,9 @@
 
 # if dcs_exists: false and dcs_type: "consul"
 [consul_instances]  # recommendation: 3 or 5-7 nodes
-#10.128.64.140 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1
-#10.128.64.142 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1
-#10.128.64.143 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1
+#10.128.64.140 consul_node_role=server consul_datacenter=dc1
+#10.128.64.142 consul_node_role=server consul_datacenter=dc1
+#10.128.64.143 consul_node_role=server consul_datacenter=dc1
 #10.128.64.144 consul_node_role=client consul_datacenter=dc2
 #10.128.64.145 consul_node_role=client consul_datacenter=dc2
 

automation/molecule/default/converge.yml

@@ -17,7 +17,6 @@
         dcs_type: "{{ 'etcd' if ansible_distribution_major_version in ['10'] or ansible_distribution_release in ['trixie']
           else (['etcd', 'consul'] | random) }}" # TODO: Consul support for RHEL 10, Debian 13
         consul_node_role: server # if dcs_type: "consul"
-        consul_bootstrap_expect: true # if dcs_type: "consul"
         postgresql_version: 18
         pgbouncer_processes: 2 # Test multiple pgbouncer processes (so_reuseport)
         patroni_tags: "datacenter=dc1,key1=value1"
@@ -61,17 +60,17 @@
 
     - name: Set variables for Extensions test
       ansible.builtin.set_fact:
-        enable_timescale: false # TODO: not available for PostgreSQL 18
+        enable_timescale: "{{ true if ansible_distribution_major_version != '10' else false }}" # TODO: not available for PostgreSQL 18 on RHEL 10
         enable_pg_repack: true
         enable_pg_cron: true
         enable_pgaudit: true
         enable_pgvector: true
         enable_postgis: true
         enable_pgrouting: true
         enable_pg_wait_sampling: true
-        enable_pg_stat_kcache: false # TODO: not available for PostgreSQL 18
-        enable_pg_partman: false # TODO: not available for PostgreSQL 18
-        enable_pgvectorscale: false # TODO: not available for PostgreSQL 18
+        enable_pg_stat_kcache: true
+        enable_pg_partman: true
+        enable_pgvectorscale: "{{ true if ansible_os_family == 'Debian' else false }}" # pgvectorscale packages are available only for Debian-based disros.
         # create extension
         postgresql_extensions:
           - { ext: "vector", db: "postgres" }

automation/molecule/pg_upgrade/converge.yml

@@ -17,7 +17,6 @@
         dcs_type: "{{ 'etcd' if ansible_distribution_major_version in ['10'] or ansible_distribution_release in ['trixie']
           else (['etcd', 'consul'] | random) }}" # TODO: Consul support for RHEL 10, Debian 13
         consul_node_role: server # if dcs_type: "consul"
-        consul_bootstrap_expect: true # if dcs_type: "consul"
         postgresql_version: 17 # redefine the version to install for the upgrade test
         pgbouncer_processes: 4 # Test multiple pgbouncer processes (so_reuseport)
         cacheable: true
@@ -27,17 +26,17 @@
     # Extension Auto-Setup
     - name: Set variables for Extensions test
       ansible.builtin.set_fact:
-        enable_timescale: false # TODO: not available for PostgreSQL 18
+        enable_timescale: "{{ true if ansible_distribution_major_version != '10' else false }}" # TODO: not available for PostgreSQL 18 on RHEL 10
         enable_pg_repack: true
         enable_pg_cron: true
         enable_pgaudit: true
         enable_pgvector: true
         enable_postgis: true
         enable_pgrouting: true
         enable_pg_wait_sampling: true
-        enable_pg_stat_kcache: false # TODO: not available for PostgreSQL 18
-        enable_pg_partman: false # TODO: not available for PostgreSQL 18
-        enable_pgvectorscale: false # TODO: not available for PostgreSQL 18
+        enable_pg_stat_kcache: true
+        enable_pg_partman: true
+        enable_pgvectorscale: "{{ true if ansible_os_family == 'Debian' else false }}" # pgvectorscale packages are available only for Debian-based disros.
         # create extension
         postgresql_extensions:
           - { ext: "vector", db: "postgres" }

automation/playbooks/consul_cluster.yml

@@ -148,14 +148,6 @@
         consul_dnsmasq_servers: "{{ consul_dnsmasq_servers | reject('equalto', '127.0.0.1') | list }}"
       when: dcs_type | default('etcd') == "consul" and consul_dnsmasq_enable | default(true) | bool and ('127.0.0.1' in (consul_dnsmasq_servers | default([])))
 
-    # Setting variables for Consul during cloud deployment
-    - name: Redefine the consul_node_role and consul_bootstrap_expect variables
-      ansible.builtin.set_fact:
-        consul_node_role: "{{ 'server' if not dcs_exists | default(false) else 'client' }}"
-        consul_bootstrap_expect: "{{ not dcs_exists | default(false) }}"
-        consul_datacenter: "{{ server_location | default('dc1') }}"
-      when: cloud_provider | default('') | length > 0
-
   roles:
     - role: vitabaks.autobase.firewall
       vars:

automation/playbooks/remove_cluster.yml

@@ -29,7 +29,7 @@
             default_postgresql_cluster_name: "{{ 'main' if ansible_os_family == 'Debian' else 'data' }}"
             default_postgresql_data_dir: "\
               {% if cloud_provider | default('') | length > 0 %}\
-              {{ pg_data_mount_path | default('/pgdata') }}/{{ default_postgresql_version }}/{{ default_postgresql_cluster_name }}\
+              {{ postgresql_data_dir_mount_path | default('/pgdata') }}/{{ default_postgresql_version }}/{{ default_postgresql_cluster_name }}\
               {% else %}\
               {{ default_postgresql_home_dir }}/{{ default_postgresql_version }}/{{ default_postgresql_cluster_name }}\
               {% endif %}"

automation/playbooks/remove_node.yml

@@ -89,7 +89,7 @@
             default_postgresql_cluster_name: "{{ 'main' if ansible_os_family == 'Debian' else 'data' }}"
             default_postgresql_data_dir: "\
               {% if cloud_provider | default('') | length > 0 %}\
-              {{ pg_data_mount_path | default('/pgdata') }}/{{ default_postgresql_version }}/{{ default_postgresql_cluster_name }}\
+              {{ postgresql_data_dir_mount_path | default('/pgdata') }}/{{ default_postgresql_version }}/{{ default_postgresql_cluster_name }}\
               {% else %}\
               {{ default_postgresql_home_dir }}/{{ default_postgresql_version }}/{{ default_postgresql_cluster_name }}\
               {% endif %}"

automation/requirements.txt

@@ -1,5 +1,5 @@
-ansible==12.1.0
-boto3==1.40.61
+ansible==12.2.0
+boto3==1.40.74
 dopy==0.3.7
-google-auth==2.42.0
-hcloud==2.9.0
+google-auth==2.43.0
+hcloud==2.11.1

automation/requirements.yml

@@ -1,24 +1,24 @@
 ---
 collections:
   - name: amazon.aws
-    version: ">=10.1.1"
+    version: ">=10.1.2"
   - name: community.aws
     version: ">=10.0.0"
   - name: google.cloud
-    version: ">=1.8.0"
+    version: ">=1.10.2"
   - name: azure.azcollection
-    version: ">=3.8.0"
+    version: ">=3.10.1"
   - name: community.digitalocean
     version: ">=1.27.0"
   - name: hetzner.hcloud
-    version: ">=5.2.0"
+    version: ">=5.4.0"
   - name: community.postgresql
-    version: ">=3.14.2"
+    version: ">=4.1.0"
   - name: community.docker
-    version: ">=4.6.1"
+    version: ">=4.8.2"
   - name: community.general
-    version: ">=10.7.2"
+    version: ">=11.4.1"
   - name: ansible.posix
-    version: ">=1.6.2"
+    version: ">=2.1.0"
   - name: ansible.utils
-    version: ">=5.1.2"
+    version: ">=6.0.0"

automation/roles/authorized_keys/tasks/main.yml

@@ -9,19 +9,17 @@
     - name: "Add public keys to ~{{ system_user.stdout | default('') }}/.ssh/authorized_keys"
       ansible.posix.authorized_key:
         user: "{{ system_user.stdout }}"
-        key: "{{ item }}"
+        key: "{{ item | replace(\"'\", '') | replace('\"', '') | trim }}"
         state: present
-      loop: '{{ ssh_public_keys_list | map(''replace'', ''"'', '''') | map(''replace'', "''", "") | list }}'
-      vars:
-        ssh_public_keys_list: >-
-          {{
-            (ssh_public_keys
-            | replace('\n', ',')
-            | split(',')
-            | map('trim')
-            | list)
-            if ssh_public_keys is string else ssh_public_keys
-          }}
+      loop: >-
+        {{
+          (ssh_public_keys
+          | replace('\n', ',')
+          | split(',')
+          | reject('equalto', '')
+          | list)
+          if ssh_public_keys is string else ssh_public_keys
+        }}
   when:
     - ssh_public_keys is defined
     - ssh_public_keys | length > 0

automation/roles/cloud_resources/defaults/main.yml

@@ -62,12 +62,14 @@ azure_blob_storage_absent: false # Allow to delete Azure Blob Storage when delet
 
 digital_ocean_spaces_create: true # if 'cloud_provider=digitalocean'
 digital_ocean_spaces_name: "{{ patroni_cluster_name }}-backup" # Name of the Spaces Object Storage (S3 bucket).
-digital_ocean_spaces_region: "nyc3" # The region to create the Space in.
+digital_ocean_spaces_region: "{{ (server_location in ['nyc1', 'nyc2']) | ternary('nyc3', server_location) }}" # The region to create the Space in.
+digital_ocean_spaces_access_key: "" # (required) Spaces Object Storage ACCESS KEY
+digital_ocean_spaces_secret_key: "" # (required) Spaces Object Storage SECRET KEY
 digital_ocean_spaces_absent: false # Allow to delete Spaces Object Storage when deleting a cluster servers using the 'state=absent' variable.
 
 hetzner_object_storage_create: true # if 'cloud_provider=hetzner'
 hetzner_object_storage_name: "{{ patroni_cluster_name }}-backup" # Name of the Object Storage (S3 bucket).
-hetzner_object_storage_region: "{{ server_location }}" # The region where the Object Storage (S3 bucket) will be created.
+hetzner_object_storage_region: "{{ (server_location in ['hel1', 'fsn1', 'nbg1']) | ternary(server_location, 'nbg1') }}" # The region where the Object Storage (S3 bucket) will be created.
 hetzner_object_storage_endpoint: "https://{{ hetzner_object_storage_region }}.your-objectstorage.com"
 hetzner_object_storage_access_key: "" # (required) Object Storage ACCESS KEY
 hetzner_object_storage_secret_key: "" # (required) Object Storage SECRET KEY

automation/roles/cloud_resources/tasks/aws.yml

@@ -351,6 +351,25 @@
           register: ec2_spot_request_result
           when: item.instances[0] | default('') | length < 1
 
+        - name: "AWS: Wait for EC2 Spot instance to be created"
+          amazon.aws.ec2_instance:
+            access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}"
+            secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}"
+            region: "{{ server_location }}"
+            filters:
+              spot-instance-request-id: "{{ item.spot_request.spot_instance_request_id }}"
+          loop: "{{ ec2_spot_request_result.results }}"
+          loop_control:
+            index_var: idx
+            label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
+          register: ec2_spot_instance_wait_result
+          until:
+            - ec2_spot_instance_wait_result.instances[0][ip_address_type] is defined
+            - ec2_spot_instance_wait_result.instances[0][ip_address_type] | length > 0
+          retries: 12
+          delay: 10
+          when: item.spot_request.spot_instance_request_id is defined
+
         - name: "AWS: Rename the EC2 Spot instance"
           amazon.aws.ec2_instance:
             access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}"
@@ -364,9 +383,7 @@
             index_var: idx
             label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}"
           register: ec2_spot_instance_result
-          until:
-            - ec2_spot_instance_result.instances[0][ip_address_type] is defined
-            - ec2_spot_instance_result.instances[0][ip_address_type] | length > 0
+          until: ec2_spot_instance_result is success
           retries: 3
           delay: 10
           when: item.spot_request.spot_instance_request_id is defined

automation/roles/cloud_resources/tasks/digitalocean.yml

@@ -143,15 +143,15 @@
       ansible.builtin.set_fact:
         default_ip_range: >-
           {{
-            vpc_info.data
+            vpc_info.get('data', [])
             | selectattr('region', 'equalto', server_location)
             | selectattr('default', 'equalto', true)
             | map(attribute='ip_range')
             | first
           }}
       when:
         - server_network | length < 1
-        - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length > 0
+        - vpc_info.get('data', []) | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length > 0
 
     # if server_network is not specified and there is no default VPC, create a network
     - name: "DigitalOcean: Create a VPC '{{ digital_ocean_vpc_name | default('network-' + server_location | default('')) }}'"
@@ -163,32 +163,41 @@
       register: digital_ocean_vpc
       when:
         - server_network | length < 1
-        - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length == 0
+        - vpc_info.get('data', []) | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length == 0
 
     - name: "Set variable: server_network"
       ansible.builtin.set_fact:
         server_network: "{{ digital_ocean_vpc_name | default('network-' + server_location) }}"
-      when: digital_ocean_vpc is changed
+      when:
+        - digital_ocean_vpc is defined
+        - digital_ocean_vpc is changed
 
     - name: "DigitalOcean: Gather information about VPC"
       community.digitalocean.digital_ocean_vpc_info:
         oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}"
       register: vpc_info
-      when: digital_ocean_vpc is changed
+      when:
+        - digital_ocean_vpc is defined
+        - digital_ocean_vpc is changed
 
     # if server_network is specified
     - name: "Fail if no VPC found in the specified region"
       ansible.builtin.fail:
         msg: "No VPC found with name '{{ server_network }}' in region '{{ server_location }}'"
       when:
         - server_network | length > 0
-        - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('name', 'equalto', server_network) | list | length == 0
+        - (vpc_info.get('data', [])
+            | selectattr('region', 'equalto', server_location)
+            | selectattr('name', 'equalto', server_network)
+            | list
+            | length
+          ) == 0
 
     - name: Extract ip_range from VPC "{{ server_network | default('') }}"
       ansible.builtin.set_fact:
         vpc_ip_range: >-
           {{
-            vpc_info.data
+            vpc_info.get('data', [])
             | selectattr('region', 'equalto', server_location)
             | selectattr('name', 'equalto', server_network)
             | map(attribute='ip_range')
@@ -200,7 +209,7 @@
       ansible.builtin.set_fact:
         vpc_id: >-
           {{
-            vpc_info.data
+            vpc_info.get('data', [])
             | selectattr('region', 'equalto', server_location)
             | selectattr('name', 'equalto', server_network)
             | map(attribute='id')
@@ -620,9 +629,12 @@
         oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}"
         name: "{{ digital_ocean_spaces_name }}"
         region: "{{ digital_ocean_spaces_region }}"
-        aws_access_key_id: "{{ AWS_ACCESS_KEY_ID }}"
-        aws_secret_access_key: "{{ AWS_SECRET_ACCESS_KEY }}"
+        aws_access_key_id: "{{ digital_ocean_spaces_access_key | default(AWS_ACCESS_KEY_ID | default(default_access_key), true) }}"
+        aws_secret_access_key: "{{ digital_ocean_spaces_secret_key | default(AWS_SECRET_ACCESS_KEY | default(default_secret_key), true) }}"
         state: present
+      vars:
+        default_access_key: "{{ pgbackrest_s3_key | default(wal_g_aws_access_key_id | default('')) }}"
+        default_secret_key: "{{ pgbackrest_s3_key_secret | default(wal_g_aws_secret_access_key | default('')) }}"
       when:
         - (pgbackrest_install | bool or wal_g_install | bool)
         - digital_ocean_spaces_create | bool
@@ -687,7 +699,7 @@
             | map(attribute='data') | map('default', []) | list | flatten
             | selectattr('id','defined') | map(attribute='id') | list
           }}
       loop: "{{ droplet_result.results | selectattr('data','defined') | list }}"
       loop_control:
         label: >-
           public_ip: {{ (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public')).0.ip_address | default('') }},