collector.yaml

kind: MetricsPrometheusCollector
prefix: kube_usage
# preComputeMetrics configures pre-computing and caching metrics
# to reduce response latency
preComputeMetrics:
  enabled: true
  frequency: "2m"
# cacheOptions configures the informer cache
cacheOptions:
  unsafeDisableDeepCopy: false
  dropAnnotations:
  - "kubectl.kubernetes.io/last-applied-configuration"
#
# utilizationServer configures how utilization metrics are pushed
# to the collector
#
utilizationServer:
  # unhealthyNodeConditions is used to determine whether we should expect samples from
  # a node.  Unhealthy nodes do not block readiness.
  unhealthyNodeConditions:
  - type: "Ready"
    status: "Unknown"
  - type: "Ready"
    status: "False"
  # protoBindPort is the port used by the node-sampler to push metrics using proto format
  protoBindPort: 9090
  # jsonProtoBindPort enables querying the grpc service with json
  jsonProtoBindPort: 8090
  # check for expired responses every 5 minutes, and remove them from the map
  expireReponsesFrequencyDuration: 5m
  # consider responses expired and stop exporting them after 5 minutes
  # this is the length of the samples window
  responseTTLDuration: 5m
  # self register all collectors with this set of node-sampler pods
  samplerPodLabels: {"app": "metrics-node-sampler"}
  samplerNamespaceName: "usage-metrics-collector"
  collectorPodLabels: {"app": "metrics-prometheus-collector"}
  # wait until we have results from 90% of healthy nodes before serving results or
  # marking ourselves as ready
  minResultPctBeforeReady: 90
  # after reaching 90% wait 2 additional registration cycles to try to catch any
  # remaining nodes.
  waitSamplerRegistrationsBeforeServe: 0
  # after reaching 90% wait 5 additional registrations after serving metrics
  # before markings ourselves as ready.
  # this gives prometheus several chances to scrape us
  waitSamplerRegistrationsBeforeReady: 0
  # delete node sampler pods if collector can't register with them after they are this old
  # this allows samplers to recover from any intermittent issues that require pod deletion
  # to remediate.
  deleteUnregisteredPodsAfterAgeMinutes: 30
  # number of inital cycles to wait before deleting node sampler pods that collector
  # can't register with.  Prevents us from deleting pods that may be old, but we just
  # connected to on startup.
  deleteUnregisteredPodsAfterCycles: 10
# config host-level metrics
cgroupMetrics:
  sources:
    "/": {name: "cgroup_utilization", avgName: "avg_cgroup_utilization"} # map a cgroup level to the corresponding metrics
  rootSource: {name: "utilization", avgName: "avg_utilization"} # aggregate node metrics
#
#
# resources defines the type of compute resources to collect metrics for
#
resources:
  cpu: "cpu_cores" # use "cpu_cores" as the metrics suffix for "cpu" resources
  memory: "memory_bytes"
  items: "items" # count of objects
#
# extensions defines extension metric labels read from Kubernetes objects
#
extensions:
  # podLabels:
  # - name: example_label_from_pod_annotation
  #   annotation: "ANNOTATION_NAME"
  # - name: example_label_from__podlabel
  #   label: "LABEL_NAME"

  # namespaceLabels:
  # - name: example_label_from_namespace_annotation
  #   annotation: "ANNOTATION_NAME"
  # - name: example_label_from_namespace_label
  #   label: "LABEL_NAME"

  # nodeLabels:
  # - name: example_label_from_node_annotation
  #   annotation: "ANNOTATION_NAME"
  # - name: example_label_from_node_label
  #   label: "LABEL_NAME"

  #
  # nodeTaints set metrics labels derived from node taints.
  #
  nodeTaints:
  # set a metric label "node_no_schedule" if no-schedule taints are present
  - name: node_no_schedule # label name
    value: true # label value if taints present
    negativeValue: false # label value if taints not present
    effects:
    - operator: In
      values: [ NoSchedule, NoExecute ]
#
# Metrics
#
aggregations:
- sources:
    type: "cgroup"
    cgroup:
    - "cgroup_utilization"
  levels:
  # kube_usage_nodecluster_hist_cgroup_utilization_cpu_cores{}
  # kube_usage_nodecluster_hist_cgroup_utilization_memory_bytes{}
  - mask:
      name: "nodecluster"
      builtIn:
        cgroup: true
      extensions:
        node_no_schedule: true
    operation: "hist"
    histogramBuckets:
        cpu_cores: [ 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 3, 4, 5, 6, 7, 8 ]
        memory_bytes:
        - 1048576 # 1MiB
        - 5242880 # 5MiB
        - 10485760 # 10MiB
        - 20971520 # 20MiB
        - 52428800 # 50MiB
        - 104857600 # 100MiB
        - 209715200 # 200MiB
        - 524288000 # 500MiB
        - 1073741824 # 1GiB
        - 2147483648 # 2GiB
        - 5368709120 # 5GiB
        - 10737418240 # 10 GiB
#
# Node Capacity Metrics
#
# Node capacity metrics aggregate node capacity, requests and limits.
# They are useful for understanding how much of each cluster's capacity
# has been consumed by running pods.
#
- sources:
    type: "node"
    node:
    - "node_capacity" # total capacity as reported by the to the apiserver
    - "node_allocatable" # schedulable capacity as reported by the to the apiserver
    - "node_requests" # total active pod requests scheduled to the node
    - "node_limits" # total active pod limits scheduled to the node
  levels:
  # kube_usage_node_sum_node_allocatable_cpu_cores{}
  # kube_usage_node_sum_node_allocatable_memory_bytes{}
  # kube_usage_node_sum_node_requests_cpu_cores{}
  # kube_usage_node_sum_node_requests_memory_bytes{}
  # kube_usage_node_sum_node_limits_cpu_cores{}
  # kube_usage_node_sum_node_limits_memory_bytes{}
  # kube_usage_node_sum_node_capacity_cpu_cores{}
  # kube_usage_node_sum_node_capacity_memory_bytes{}
  - mask:
      name: "node"
      builtIn:
        exported_node: true
      extensions:
        node_no_schedule: true
    operation: "sum"
  # kube_usage_nodecluster_sum_node_allocatable_cpu_cores{}
  # kube_usage_nodecluster_sum_node_allocatable_memory_bytes{}
  # kube_usage_nodecluster_sum_node_requests_cpu_cores{}
  # kube_usage_nodecluster_sum_node_requests_memory_bytes{}
  # kube_usage_nodecluster_sum_node_limits_cpu_cores{}
  # kube_usage_nodecluster_sum_node_limits_memory_bytes{}
  # kube_usage_nodecluster_sum_node_capacity_cpu_cores{}
  # kube_usage_nodecluster_sum_node_capacity_memory_bytes{}
  - mask:
      name: "nodecluster"
      extensions:
        node_no_schedule: true
    operation: "sum"
- sources:
    type: "cgroup"
    cgroup:
    - "avg_cgroup_utilization"
    alias:
      "avg_cgroup_utilization": "cgroup_utilization"
  levels:
  # kube_usage_node_sum_utilization_cpu_cores{}
  # kube_usage_node_sum_utilization_memory_bytes{}
  - mask:
      name: "node"
      builtIn:
        exported_node: true
      extensions:
        node_no_schedule: true
    operation: "sum"
  # kube_usage_nodecluster_sum_utilization_cpu_cores{}
  # kube_usage_nodecluster_sum_utilization_memory_bytes{}
  - mask:
      name: "nodecluster"
      extensions:
        node_no_schedule: true
    operation: "sum"
#
# Workload Utilization Metrics
#
- sources:
    type: "container"
    container: [ "utilization" ]
  levels:
  # kube_usage_workload_avg_utilization_cpu_cores{}
  # kube_usage_workload_avg_utilization_memory_bytes{}
  - mask:
      name: "workload"
      builtIn:
        exported_container: true  # keep the individual container names within each pod, but not the pod names
        exported_namespace: true
        workload_name: true
        workload_kind: true
        workload_api_group: true
        workload_api_version: true
        app: true
        priority_class: true
    operations: [ "avg", "median", "p95", "max" ]
    retentionName: container
#
# Workload Requests and Limits Metrics
#
- sources:
    type: "container"
    container: [ "requests_allocated", "limits_allocated" ]
  levels:
  # kube_usage_workload_avg_requests_allocated_cpu_cores{}
  # kube_usage_workload_avg_limits_allocated_memory_bytes{}
  - mask:
      name: "workload"
      builtIn:
        exported_container: true  # keep the individual container names within each pod, but not the pod names
        exported_namespace: true
        workload_name: true
        workload_kind: true
        workload_api_group: true
        workload_api_version: true
        app: true
        priority_class: true
    operations: [ "avg" ]
    retentionName: container
#
# Workload Capacity Metrics
#
# Workload capacity metrics aggregate the resources consumed by containers as
# requests / limits in the cluster.
#
- sources:
    type: "container"
    container: [ "requests_allocated", "limits_allocated", "avg_utilization" ]
    alias:
      "avg_utilization": "utilization"
  levels:
  # kube_usage_workload_sum_utilization_cpu_cores{}
  # kube_usage_workload_sum_utilization_memory_bytes{}
  # kube_usage_workload_sum_requests_allocated_cpu_cores{}
  # kube_usage_workload_sum_requests_allocated_memory_bytes{}
  # kube_usage_workload_sum_limits_allocated_cpu_cores{}
  # kube_usage_workload_sum_limits_allocated_memory_bytes{}
  - mask:
      name: "workload"
      builtIn:
        exported_namespace: true
        workload_name: true
        workload_kind: true
        workload_api_group: true
        workload_api_version: true
        app: true
        priority_class: true
    operation: "sum"
    retentionName: workload
  # kube_usage_namespace_sum_utilization_cpu_cores{}
  # kube_usage_namespace_sum_utilization_memory_bytes{}
  # kube_usage_namespace_sum_requests_allocated_cpu_cores{}
  # kube_usage_namespace_sum_requests_allocated_memory_bytes{}
  # kube_usage_namespace_sum_limits_allocated_cpu_cores{}
  # kube_usage_namespace_sum_limits_allocated_memory_bytes{}
  - mask:
      name: "namespace"
      builtIn:
        exported_namespace: true
        priority_class: true
    operation: "sum"
  # kube_usage_cluster_sum_utilization_cpu_cores{}
  # kube_usage_cluster_sum_utilization_memory_bytes{}
  # kube_usage_cluster_sum_requests_allocated_cpu_cores{}
  # kube_usage_cluster_sum_requests_allocated_memory_bytes{}
  # kube_usage_cluster_sum_limits_allocated_cpu_cores{}
  # kube_usage_cluster_sum_limits_allocated_memory_bytes{}
  - mask:
      name: "cluster"
      builtIn:
        priority_class: true
    operation: "sum"
#
# Namespace Quota Metrics
#
# Namespace quota metrics aggregate the requests / limits quota for each namespace.
#
- sources:
    type: "quota"
    quota:
    - "requests_quota_hard"
    - "requests_quota_used"
    - "limits_quota_hard"
    - "limits_quota_used"
  levels:
  # kube_usage_namespace_sum_requests_quota_hard_cpu_cores{}
  # kube_usage_namespace_sum_requests_quota_hard_memory_bytes{}
  # kube_usage_namespace_sum_requests_quota_used_cpu_cores{}
  # kube_usage_namespace_sum_requests_quota_used_memory_bytes{}
  # kube_usage_namespace_sum_limits_quota_hard_cpu_cores{}
  # kube_usage_namespace_sum_limits_quota_hard_memory_bytes{}
  # kube_usage_namespace_sum_limits_quota_used_cpu_cores{}
  # kube_usage_namespace_sum_limits_quota_used_memory_bytes{}
  - mask:
      name: "namespace"
      builtIn:
        exported_namespace: true
        priority_class: true
        storage_class: true
    operation: "sum"
  # kube_usage_cluster_sum_requests_quota_hard_cpu_cores{}
  # kube_usage_cluster_sum_requests_quota_hard_memory_bytes{}
  # kube_usage_cluster_sum_requests_quota_used_cpu_cores{}
  # kube_usage_cluster_sum_requests_quota_used_memory_bytes{}
  # kube_usage_cluster_sum_limits_quota_hard_cpu_cores{}
  # kube_usage_cluster_sum_limits_quota_hard_memory_bytes{}
  # kube_usage_cluster_sum_limits_quota_used_cpu_cores{}
  # kube_usage_cluster_sum_limits_quota_used_memory_bytes{}
  - mask:
      name: "cluster"
      builtIn:
        priority_class: true
        storage_class: true
    operation: "sum"
# saveSamplesLocally:
#   directoryPath: /metrics-prometheus-collector-samples
#   timeFormat: "20060102/15:04:05"
#   saveProto: false
#   saveJSON: true