add variables to control autoscaler

Author: streamnsight

add_on_k8s_autoscaler.tf

@@ -7,10 +7,10 @@ locals {
   cluster_autoscaler_image_version                    = lookup(local.cluster_autoscaler_supported_k8s_versions, local.k8s_major_minor_version, reverse(values(local.cluster_autoscaler_supported_k8s_versions))[0])
   cluster_autoscaler_image                            = "iad.ocir.io/oracle/oci-cluster-autoscaler:${local.cluster_autoscaler_image_version}"
   cluster_autoscaler_log_level_verbosity              = 4
-  cluster_autoscaler_max_node_provision_time          = "25m"
-  cluster_autoscaler_scale_down_delay_after_add       = "10m"
-  cluster_autoscaler_scale_down_unneeded_time         = "10m"
-  cluster_autoscaler_unremovable_node_recheck_timeout = "5m"
+  cluster_autoscaler_max_node_provision_time          = "${var.cluster_autoscaler_max_node_provision_time}m"
+  cluster_autoscaler_scale_down_delay_after_add       = "${var.cluster_autoscaler_scale_down_delay_after_add}m"
+  cluster_autoscaler_scale_down_unneeded_time         = "${var.cluster_autoscaler_scale_down_unneeded_time}m"
+  cluster_autoscaler_unremovable_node_recheck_timeout = "${var.cluster_autoscaler_unremovable_node_recheck_timeout}m"
   cluster_autoscaler_cloud_provider                   = tonumber(local.k8s_minor_version) <= 23 ? "oci" : "oci-oke"
   cluster_autoscaler_enabled                          = contains(keys(local.cluster_autoscaler_supported_k8s_versions), local.k8s_major_minor_version) ? (var.np1_enable_autoscaler || var.np2_enable_autoscaler || var.np3_enable_autoscaler) : false
   k8s_major_minor_version                             = regex("\\d+(?:\\.(?:\\d+|x)(?:))", local.kubernetes_version)
@@ -273,9 +273,9 @@ resource "kubernetes_deployment" "cluster_autoscaler_deployment" {
             "--stderrthreshold=info",
             "--cloud-provider=${local.cluster_autoscaler_cloud_provider}",
             "--max-node-provision-time=${local.cluster_autoscaler_max_node_provision_time}",
-            "--nodes=${var.np1_autoscaler_min_nodes}:${var.np1_autoscaler_max_nodes}:${oci_containerengine_node_pool.oci_oke_node_pool[0].id}",
-            var.node_pool_count >= 2 ? "--nodes=${var.np2_autoscaler_min_nodes}:${var.np2_autoscaler_max_nodes}:${oci_containerengine_node_pool.oci_oke_node_pool[1].id}" : "",
-            var.node_pool_count >= 3 ? "--nodes=${var.np3_autoscaler_min_nodes}:${var.np3_autoscaler_max_nodes}:${oci_containerengine_node_pool.oci_oke_node_pool[2].id}" : "",
+            var.np1_enable_autoscaler ? "--nodes=${var.np1_autoscaler_min_nodes}:${var.np1_autoscaler_max_nodes}:${oci_containerengine_node_pool.oci_oke_node_pool[0].id}" : "",
+            var.node_pool_count >= 2 && var.np2_enable_autoscaler ? "--nodes=${var.np2_autoscaler_min_nodes}:${var.np2_autoscaler_max_nodes}:${oci_containerengine_node_pool.oci_oke_node_pool[1].id}" : "",
+            var.node_pool_count >= 3 && var.np3_enable_autoscaler ? "--nodes=${var.np3_autoscaler_min_nodes}:${var.np3_autoscaler_max_nodes}:${oci_containerengine_node_pool.oci_oke_node_pool[2].id}" : "",
             "--scale-down-delay-after-add=${local.cluster_autoscaler_scale_down_delay_after_add}",
             "--scale-down-unneeded-time=${local.cluster_autoscaler_scale_down_unneeded_time}",
             "--unremovable-node-recheck-timeout=${local.cluster_autoscaler_unremovable_node_recheck_timeout}",
@@ -338,3 +338,28 @@ resource "kubernetes_pod_disruption_budget_v1" "core_dns_pod_disruption_budget"
     oci_containerengine_node_pool.oci_oke_node_pool
   ]
 }
+
+resource "kubernetes_pod_disruption_budget_v1" "cluster_autoscaler_pod_disruption_budget" {
+  count = local.cluster_autoscaler_enabled ? 1 : 0
+
+  metadata {
+    name      = "cluster-autoscaler-pdb"
+    namespace = "kube-system"
+    labels = {
+      k8s-app = "cluster-autoscaler"
+    }
+  }
+  spec {
+    max_unavailable = "1"
+    selector {
+      match_labels = {
+        app = "cluster-autoscaler"
+      }
+    }
+  }
+
+  depends_on = [
+    data.oci_containerengine_cluster_kube_config.oke,
+    oci_containerengine_node_pool.oci_oke_node_pool
+  ]
+}

oke_cluster.tf

@@ -2,7 +2,7 @@
 ## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
 
 locals {
-  kubernetes_version = var.kubernetes_version != "" ? var.kubernetes_version : reverse(data.oci_containerengine_cluster_option.cluster_options.kubernetes_versions)[0]
+  kubernetes_version = (var.kubernetes_version != "" && var.kubernetes_version != null) ? var.kubernetes_version : reverse(data.oci_containerengine_cluster_option.cluster_options.kubernetes_versions)[0]
 }
 
 resource "oci_containerengine_cluster" "oci_oke_cluster" {

oke_node_pools.tf

@@ -57,8 +57,8 @@ resource "oci_containerengine_node_pool" "oci_oke_node_pool" {
 
   cluster_id         = oci_containerengine_cluster.oci_oke_cluster.id
   compartment_id     = var.cluster_compartment_id
-  kubernetes_version = var.kubernetes_version != "" ? var.kubernetes_version : reverse(data.oci_containerengine_cluster_option.cluster_options.kubernetes_versions)[0]
-  name               = "${local.node_pools[count.index]["node_shape"]}_Node_Pool"
+  kubernetes_version = (var.kubernetes_version != "" && var.kubernetes_version != null) ? var.kubernetes_version : reverse(data.oci_containerengine_cluster_option.cluster_options.kubernetes_versions)[0]
+  name               = "${replace(local.node_pools[count.index]["node_shape"], "Standard", "Std")}${length(regexall("Flex", local.node_pools[count.index]["node_shape"])) > 0 ? "-${local.node_pools[count.index]["ocpus"]}-${local.node_pools[count.index]["memory_gb"]}GB" : ""}"
   node_shape         = local.node_pools[count.index]["node_shape"]
 
   #   initial_node_labels {

schema.yaml

@@ -156,6 +156,18 @@ variableGroups:
     variables:
       - enable_pod_admission_controller
 
+  - title: "Cluster Autoscaler Settings"
+    variables:
+      - cluster_autoscaler_max_node_provision_time
+      - cluster_autoscaler_scale_down_delay_after_add
+      - cluster_autoscaler_scale_down_unneeded_time
+      - cluster_autoscaler_unremovable_node_recheck_timeout
+    visible:
+      or: 
+      - np1_enable_autoscaler
+      - np2_enable_autoscaler
+      - np3
+
   - title: "Deployments"
     variables:
       - enable_flink
@@ -1079,6 +1091,22 @@ variables:
     description: |
       Deploys Prometheus, Grafana and related datasources, plugins and dashboards.
 
+  cluster_autoscaler_max_node_provision_time:
+    type: number
+    title: Max node provisioning time before it is considered failed.
+
+  cluster_autoscaler_scale_down_delay_after_add:
+    type: number
+    title: Wait time before scaling a node down after it was added.
+
+  cluster_autoscaler_scale_down_unneeded_time:
+    type: number
+    title: Wait time before scaling a node down after it is unneeded.
+
+  cluster_autoscaler_unremovable_node_recheck_timeout:
+    type: number
+    title: Wait time between checks on unremovable nodes.
+
 outputGroups:
   - title: Access
     outputs: 
@@ -1117,4 +1145,4 @@ outputs:
     type: copyableString
     title: Grafana access
     displayText: "Grafana password for 'admin' user"
-    visible: 
+    visible: enable_monitoring_stack

terraform.tfvars.template

@@ -2,11 +2,11 @@
 ## All rights reserved. The Universal Permissive License (UPL), Version 1.0 as shown at http://oss.oracle.com/licenses/upl
 
 region       = "us-ashburn-1"
-tenancy_ocid = "ocid1.tenancy.oc1..."
+tenancy_ocid = "ocid1.tenancy.oc1.."
 
 ## Compartments
-vcn_compartment_id     = "ocid1.compartment.oc1..."
-cluster_compartment_id = "ocid1.compartment.oc1..."
+vcn_compartment_id     = "ocid1.compartment.oc1.."
+cluster_compartment_id = "ocid1.compartment.oc1.."
 
 ## Network
 use_existing_vcn = false
@@ -17,7 +17,7 @@ is_endpoint_public = true
 
 ## Cluster
 cluster_name = "Flink Cluster"
-# kubernetes_version=
+kubernetes_version=""
 ssh_public_key  = "ssh-rsa AAAA..."
 node_pool_count = 1
 # add_cluster_tag=
@@ -76,3 +76,4 @@ image_validation_key_id = null # "ocid1.key.oc1..."
 enable_cert_manager = true
 enable_flink = true
 enable_metrics_server = true
+enable_monitoring_stack = true

variables.tf

@@ -38,7 +38,7 @@ variable "cluster_name" {
 }
 
 variable "kubernetes_version" {
-  default = ""
+  default = null
   # default to latest version if null
 }
 
@@ -277,4 +277,21 @@ variable "enable_flink" {
 
 variable "enable_monitoring_stack" {
   default = true
+}
+
+variable "cluster_autoscaler_max_node_provision_time" {
+  default = 25
+  description = "Maximum wait time (min) for nodes to provision before failure"
+}
+variable cluster_autoscaler_scale_down_delay_after_add { 
+  default = 10
+  description = "Minimum delay (min) before scaling a node down after it was provisioned"
+} 
+variable cluster_autoscaler_scale_down_unneeded_time { 
+  default = 10 
+  description = "Minimum delay (min) before scaling a node down once it is unneeded"
+}
+variable cluster_autoscaler_unremovable_node_recheck_timeout { 
+  default = 5 
+  description = "Time (min) between checks on status of unremovable"
 }