Improve Patroni leader health check logic (#1306)

Updated the health check to fail only if no healthy Patroni leader node is detected, using a more robust condition. The error message now provides clearer instructions for troubleshooting cluster status and REST API connectivity.

Author: vitabaks

automation/playbooks/config_pgcluster.yml

@@ -66,14 +66,19 @@
         no_proxy: "{{ patroni_bind_address | default(bind_address, true) }}"
       when: inventory_hostname in groups['postgres_cluster']
 
-    # Stop, if Patroni is unavailable
+    # Stop, if no healthy Patroni leader node
     - name: The Patroni cluster is unhealthy
       ansible.builtin.fail:
-        msg: "Patroni is unavailable on {{ ansible_hostname }}. Please check the cluster status."
-      changed_when: false
+        msg: >
+          No healthy leader node detected in cluster '{{ patroni_cluster_name | default('postgres-cluster') }}'.
+          Please ensure the cluster is up and node responds on REST API port ({{ patroni_restapi_port | default('8008') }}).
       when:
-        - inventory_hostname in groups['postgres_cluster']
-        - (patroni_leader_result is undefined or patroni_leader_result.status == -1)
+        - inventory_hostname == groups['postgres_cluster'][0]
+        - (hostvars
+            | dict2items
+            | selectattr('value.patroni_leader_result.status', 'defined')
+            | selectattr('value.patroni_leader_result.status', 'equalto', 200)
+            | list | length) == 0
 
   roles:
     - role: vitabaks.autobase.pre_checks
@@ -98,7 +103,7 @@
         name: "{{ item }}"
         groups: secondary
         postgresql_exists: true
-      when: hostvars[item]['patroni_leader_result']['status'] != 200
+      when: hostvars[item]['patroni_leader_result']['status'] == 503
       loop: "{{ groups['postgres_cluster'] }}"
       changed_when: false
       check_mode: false
@@ -108,7 +113,7 @@
         msg:
           - "Cluster Name: {{ patroni_cluster_name | default('postgres-cluster') }}"
           - "Cluster Leader: {{ ansible_hostname }}"
-      when: inventory_hostname in groups['primary']
+      when: inventory_hostname in groups['primary'] | default([])
 
     # if 'cloud_provider' is 'aws', 'gcp', or 'azure'
     # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI)

automation/playbooks/pg_upgrade.yml

@@ -30,15 +30,23 @@
       register: patroni_leader_result
       changed_when: false
       failed_when: false
+      check_mode: false
       environment:
         no_proxy: "{{ patroni_bind_address | default(bind_address, true) }}"
 
-    # Stop, if Patroni is unavailable
+    # Stop, if no healthy Patroni leader node
     - name: The Patroni cluster is unhealthy
       ansible.builtin.fail:
-        msg: "Patroni is unavailable on {{ ansible_hostname }}. Please check the cluster status."
-      changed_when: false
-      when: patroni_leader_result is undefined or patroni_leader_result.status == -1
+        msg: >
+          No healthy leader node detected in cluster '{{ patroni_cluster_name | default('postgres-cluster') }}'.
+          Please ensure the cluster is up and node responds on REST API port ({{ patroni_restapi_port | default('8008') }}).
+      when:
+        - inventory_hostname == groups['postgres_cluster'][0]
+        - (hostvars
+            | dict2items
+            | selectattr('value.patroni_leader_result.status', 'defined')
+            | selectattr('value.patroni_leader_result.status', 'equalto', 200)
+            | list | length) == 0
 
     - name: '[Prepare] Add host to group "primary" (in-memory inventory)'
       ansible.builtin.add_host:
@@ -52,7 +60,7 @@
       ansible.builtin.add_host:
         name: "{{ item }}"
         groups: secondary
-      when: hostvars[item]['patroni_leader_result']['status'] != 200
+      when: hostvars[item]['patroni_leader_result']['status'] == 503
       loop: "{{ groups['postgres_cluster'] }}"
       changed_when: false
 
@@ -61,7 +69,7 @@
         msg:
           - "Cluster Name: {{ patroni_cluster_name | default('postgres-cluster') }}"
           - "Cluster Leader: {{ ansible_hostname }}"
-      when: inventory_hostname in groups['primary']
+      when: inventory_hostname in groups['primary'] | default([])
   tags:
     - always
 

automation/playbooks/update_pgcluster.yml

@@ -38,6 +38,21 @@
         no_proxy: "{{ patroni_bind_address | default(bind_address, true) }}"
       tags: always
 
+    # Stop, if no healthy Patroni leader node
+    - name: The Patroni cluster is unhealthy
+      ansible.builtin.fail:
+        msg: >
+          No healthy leader node detected in cluster '{{ patroni_cluster_name | default('postgres-cluster') }}'.
+          Please ensure the cluster is up and node responds on REST API port ({{ patroni_restapi_port | default('8008') }}).
+      when:
+        - inventory_hostname == groups['postgres_cluster'][0]
+        - (hostvars
+            | dict2items
+            | selectattr('value.patroni_leader_result.status', 'defined')
+            | selectattr('value.patroni_leader_result.status', 'equalto', 200)
+            | list | length) == 0
+      tags: always
+
     - name: '[Prepare] Add host to group "primary" (in-memory inventory)'
       ansible.builtin.add_host:
         name: "{{ item }}"
@@ -51,7 +66,7 @@
       ansible.builtin.add_host:
         name: "{{ item }}"
         groups: secondary
-      when: hostvars[item]['patroni_leader_result']['status'] != 200
+      when: hostvars[item]['patroni_leader_result']['status'] == 503
       loop: "{{ groups['postgres_cluster'] }}"
       changed_when: false
       tags: always
@@ -61,7 +76,7 @@
         msg:
           - "Cluster Name: {{ patroni_cluster_name | default('postgres-cluster') }}"
           - "Cluster Leader: {{ ansible_hostname }}"
-      when: inventory_hostname in groups['primary']
+      when: inventory_hostname in groups['primary'] | default([])
       tags: always
 
 - name: "(1/4) PRE-UPDATE: Perform pre-update tasks"