[PLAT-16454] Configuring HA alert frequency based on config

Summary: HA alert for standby down defaults to 15 minutes and doesn't change unless configured by user. This diff replaces the threshold value according to the HA config replication frequency, which is also just a runtime config value we update. We set the threshold to twice the replication frequency or 15 minutes, whichever is greater.

Test Plan: Configure HA, change the HA config frequency and ensure alert threshold also changes.

Reviewers: amalyshev

Reviewed By: amalyshev

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D41363

Author: mchiddy

managed/src/main/java/com/yugabyte/yw/common/alerts/AlertConfigurationService.java

@@ -10,6 +10,7 @@
 package com.yugabyte.yw.common.alerts;
 
 import static com.yugabyte.yw.common.Util.doubleToString;
+import static com.yugabyte.yw.common.ha.PlatformReplicationHelper.REPLICATION_FREQUENCY_KEY;
 import static com.yugabyte.yw.models.AlertConfiguration.createQueryByFilter;
 import static com.yugabyte.yw.models.helpers.CommonUtils.nowWithoutMillis;
 import static com.yugabyte.yw.models.helpers.CommonUtils.performPagedQuery;
@@ -374,6 +375,22 @@ private AlertConfiguration validate(
                             + doubleToString(templateDescription.getThresholdMaxValue()))
                     .throwError();
               }
+              if (configuration.getTemplate() == AlertTemplate.HA_STANDBY_SYNC) {
+                long replicationFrequency =
+                    runtimeConfigFactory
+                        .globalRuntimeConf()
+                        .getDuration(REPLICATION_FREQUENCY_KEY)
+                        .toMillis();
+                // threshold in minutes RF in milliseconds
+                if ((threshold.getThreshold() * 60.0) < (2.0 * replicationFrequency / 1000.0)) {
+                  beanValidator
+                      .error()
+                      .forField(
+                          "HA Standby Sync[" + severity.name() + "].threshold",
+                          "can't be less than twice replication frequency")
+                      .throwError();
+                }
+              }
             });
     if (before != null) {
       if (!configuration.getCustomerUUID().equals(before.getCustomerUUID())) {

managed/src/main/java/com/yugabyte/yw/common/ha/PlatformReplicationHelper.java

@@ -17,20 +17,25 @@
 import com.typesafe.config.ConfigException;
 import com.typesafe.config.ConfigValue;
 import com.typesafe.config.ConfigValueFactory;
+import com.yugabyte.yw.common.AlertTemplate;
 import com.yugabyte.yw.common.AppConfigHelper;
 import com.yugabyte.yw.common.PrometheusConfigHelper;
 import com.yugabyte.yw.common.PrometheusConfigManager;
 import com.yugabyte.yw.common.ShellProcessHandler;
 import com.yugabyte.yw.common.ShellResponse;
 import com.yugabyte.yw.common.SwamperHelper;
+import com.yugabyte.yw.common.alerts.AlertConfigurationService;
 import com.yugabyte.yw.common.config.GlobalConfKeys;
 import com.yugabyte.yw.common.config.RuntimeConfGetter;
 import com.yugabyte.yw.common.config.impl.SettableRuntimeConfigFactory;
 import com.yugabyte.yw.common.ha.PlatformReplicationManager.PlatformBackupParams;
 import com.yugabyte.yw.common.utils.FileUtils;
 import com.yugabyte.yw.metrics.MetricUrlProvider;
+import com.yugabyte.yw.models.AlertConfiguration;
+import com.yugabyte.yw.models.AlertConfigurationThreshold;
 import com.yugabyte.yw.models.HighAvailabilityConfig;
 import com.yugabyte.yw.models.PlatformInstance;
+import com.yugabyte.yw.models.filters.AlertConfigurationFilter;
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
@@ -47,6 +52,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.pekko.actor.Cancellable;
 import org.apache.velocity.Template;
@@ -69,7 +75,7 @@ public class PlatformReplicationHelper {
   private static final String REPLICATION_SCHEDULE_ENABLED_KEY =
       "yb.ha.replication_schedule_enabled";
   private static final String NUM_BACKUP_RETENTION_KEY = "yb.ha.num_backup_retention";
-  static final String REPLICATION_FREQUENCY_KEY = "yb.ha.replication_frequency";
+  public static final String REPLICATION_FREQUENCY_KEY = "yb.ha.replication_frequency";
   static final String DB_USERNAME_CONFIG_KEY = "db.default.username";
   static final String DB_PASSWORD_CONFIG_KEY = "db.default.password";
   static final String DB_HOST_CONFIG_KEY = "db.default.host";
@@ -92,6 +98,8 @@ public class PlatformReplicationHelper {
 
   private final PrometheusConfigManager prometheusConfigManager;
 
+  private final AlertConfigurationService alertConfigurationService;
+
   @VisibleForTesting ShellProcessHandler shellProcessHandler;
 
   @Inject
@@ -102,14 +110,16 @@ public PlatformReplicationHelper(
       ShellProcessHandler shellProcessHandler,
       MetricUrlProvider metricUrlProvider,
       PrometheusConfigHelper prometheusConfigHelper,
-      PrometheusConfigManager prometheusConfigManager) {
+      PrometheusConfigManager prometheusConfigManager,
+      AlertConfigurationService alertConfigurationService) {
     this.confGetter = confGetter;
     this.runtimeConfigFactory = runtimeConfigFactory;
     this.remoteClientFactory = remoteClientFactory;
     this.shellProcessHandler = shellProcessHandler;
     this.metricUrlProvider = metricUrlProvider;
     this.prometheusConfigHelper = prometheusConfigHelper;
     this.prometheusConfigManager = prometheusConfigManager;
+    this.alertConfigurationService = alertConfigurationService;
   }
 
   Path getBackupDir() {
@@ -203,6 +213,26 @@ public void setReplicationFrequency(Duration duration) {
     runtimeConfigFactory
         .globalRuntimeConf()
         .setValue(REPLICATION_FREQUENCY_KEY, String.format("%d ms", duration.toMillis()));
+    List<AlertConfiguration> haStandbyAlertConfigs =
+        alertConfigurationService.list(
+            AlertConfigurationFilter.builder().template(AlertTemplate.HA_STANDBY_SYNC).build());
+    haStandbyAlertConfigs.forEach(
+        haStandbyAlertConfig -> {
+          haStandbyAlertConfig.setThresholds(
+              haStandbyAlertConfig.getThresholds().entrySet().stream()
+                  .collect(
+                      Collectors.toMap(
+                          Map.Entry::getKey,
+                          e ->
+                              new AlertConfigurationThreshold()
+                                  .setCondition(e.getValue().getCondition())
+                                  .setThreshold(
+                                      Math.max(
+                                          // Convert to minutes
+                                          2 * duration.toMillis() / 1000.0 / 60.0,
+                                          e.getValue().getThreshold())))));
+          alertConfigurationService.save(haStandbyAlertConfig);
+        });
   }
 
   JsonNode getBackupInfoJson(long frequency, boolean isRunning) {

managed/src/main/resources/alert/alert_templates.yml

@@ -87,15 +87,15 @@ templates:
     createForNewCustomer: true
     defaultThresholdMap:
       WARNING:
-        threshold: 900.0
+        threshold: 15
     targetType: PLATFORM
     defaultThresholdCondition: GREATER_THAN
-    defaultThresholdUnit: SECOND
+    defaultThresholdUnit: MINUTE
     thresholdReadOnly: false
-    thresholdUnitName: seconds
+    thresholdUnitName: min
     labels:
       affected_instance_addrs: >-
-        {{ range $index, $element := query "count by (instance_address) (time() - last_over_time(yba_ha_last_backup_seconds[1m]) > {{ query_threshold }}) > 0"}}
+        {{ range $index, $element := query "count by (instance_address) ((time() - last_over_time(yba_ha_last_backup_seconds[1m]) / 60) > {{ query_threshold }}) > 0"}}
         {{ if $index }},{{ end }}{{ $element.Labels.instance_address }}{{ end }}
     annotations:
       summary: >-
@@ -1293,15 +1293,15 @@ templates:
     thresholdUnitName: ms
     labels:
       affected_node_names: >-
-        {{ range $index, $element := query 
+        {{ range $index, $element := query
         "max by (universe_uuid, node_name)
-        (max_over_time(consumer_safe_time_lag{universe_uuid='{{ $labels.universe_uuid }}'}[10m]) 
+        (max_over_time(consumer_safe_time_lag{universe_uuid='{{ $labels.universe_uuid }}'}[10m])
         {{ query_condition }} {{ query_threshold }})" }}
         {{if $index}},{{end}}{{ $element.Labels.node_name }}{{ end }}
       affected_namespace_ids: >-
-        {{ range $index, $element := query 
+        {{ range $index, $element := query
         "max by (universe_uuid, namespace_id)
-        (max_over_time(consumer_safe_time_lag{universe_uuid='{{ $labels.universe_uuid }}'}[10m]) 
+        (max_over_time(consumer_safe_time_lag{universe_uuid='{{ $labels.universe_uuid }}'}[10m])
         {{ query_condition }} {{ query_threshold }})" }}
         {{if $index}},{{end}}{{ $element.Labels.namespace_id }}{{ end }}
     annotations:

managed/src/main/resources/reference.conf

@@ -768,7 +768,7 @@ yb {
     num_backup_retention = 10
     prometheus_config_dir = "/prometheus_configs"
     replication_schedule_enabled = false
-    replication_frequency = 30 minutes
+    replication_frequency = 1 minutes
     # 0 - never shutdown
     # 1 - only shutdown promoted instance
     # 2 - shutdown both promoted and demoted instance

managed/src/test/resources/dev.expected.conf

@@ -107,7 +107,7 @@ yb.docker.release = ""
 yb.grafana.accessKey = changeme
 yb.ha.num_backup_retention = 10
 yb.ha.prometheus_config_dir = "/etc/prometheus"
-yb.ha.replication_frequency = "30 minutes"
+yb.ha.replication_frequency = "1 minutes"
 yb.ha.replication_schedule_enabled = false
 yb.health.check_interval_ms = 300000
 yb.health.default_email = "RESOLVED_YB_ALERTS_EMAIL"

managed/src/test/resources/helm.expected.conf

@@ -108,7 +108,7 @@ yb.docker.network = bridge
 yb.docker.release = "/opt/yugabyte/release"
 yb.ha.num_backup_retention = 10
 yb.ha.prometheus_config_dir = "/prometheus_configs"
-yb.ha.replication_frequency = "30 minutes"
+yb.ha.replication_frequency = "1 minutes"
 yb.ha.replication_schedule_enabled = false
 yb.health.check_interval_ms = 300000
 yb.health.default_email = "[email protected]"

managed/src/test/resources/replicated.expected.conf

@@ -106,7 +106,7 @@ yb.docker.network = bridge
 yb.docker.release = "/opt/yugabyte/release"
 yb.ha.num_backup_retention = 10
 yb.ha.prometheus_config_dir = "/prometheus_configs"
-yb.ha.replication_frequency = "30 minutes"
+yb.ha.replication_frequency = "1 minutes"
 yb.ha.replication_schedule_enabled = false
 yb.health.check_interval_ms = 300000
 yb.health.default_email = "[email protected]"

managed/src/test/resources/test.expected.conf

@@ -78,7 +78,7 @@ yb.aws.default_volume_count = 1
 yb.aws.default_volume_size_gb = 250
 yb.ha.num_backup_retention = 10
 yb.ha.prometheus_config_dir = "/prometheus_configs"
-yb.ha.replication_frequency = "30 minutes"
+yb.ha.replication_frequency = "1 minutes"
 yb.ha.replication_schedule_enabled = false
 yb.health.default_email = ""
 yb.metrics.host = localhost

managed/src/test/resources/yugabundle.expected.conf

@@ -75,7 +75,7 @@ yb.aws.default_volume_count = 1
 yb.aws.default_volume_size_gb = 250
 yb.ha.num_backup_retention = 10
 yb.ha.prometheus_config_dir = "/prometheus_configs"
-yb.ha.replication_frequency = "30 minutes"
+yb.ha.replication_frequency = "1 minutes"
 yb.ha.replication_schedule_enabled = false
 yb.health.default_email = "RESOLVED_YB_ALERTS_EMAIL"
 yb.metrics.host = localhost

managed/src/test/resources/yugabyted.expected.conf

@@ -83,7 +83,7 @@ yb.aws.default_volume_count = 1
 yb.aws.default_volume_size_gb = 250
 yb.ha.num_backup_retention = 10
 yb.ha.prometheus_config_dir = "/prometheus_configs"
-yb.ha.replication_frequency = "30 minutes"
+yb.ha.replication_frequency = "1 minutes"
 yb.ha.replication_schedule_enabled = false
 yb.health.default_email = ""
 yb.metrics.host = localhost