Reduce apiserver metric cardinality (prometheus-operator#2531)

* Reduce apiserver metric cardinality Current releases of Kubernetes added new metrics that also have the same excessive bucket cardinality. Remove the extra buckets from these metrics. * `apiserver_request_sli_duration_seconds_bucket` * `etcd_request_duration_seconds_bucket` Also reduce the cardinality of apiserver byte histogram metrics. * `apiserver_request_bdy_size_bytes_bucket` This reduces API server metrics in our configuration by about 25%. Signed-off-by: SuperQ <[email protected]> * Apply suggestions from code review --------- Signed-off-by: SuperQ <[email protected]> Co-authored-by: Simon Pasquier <[email protected]>
GrantFleming · Oct 24, 2024 · 69d9636 · 69d9636
1 parent 35a017a
commit 69d9636
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
@@ -323,7 +323,12 @@ function(params) {
             },
             {
               sourceLabels: ['__name__', 'le'],
-              regex: 'apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)',
+              regex: '(apiserver_request|apiserver_request_sli|etcd_request)_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)',
+              action: 'drop',
+            },
+            {
+              sourceLabels: ['__name__', 'le'],
+              regex: 'apiserver_request_body_size_bytes_bucket;(150000|350000|550000|650000|850000|950000|(1\\.15|1\\.35|1\\.55|1\\.65|1\\.85|1\\.95|2\\.15|2\\.35|2\\.55|2\\.65|2\\.85|2\\.95)e\\+06)',
               action: 'drop',
             },
           ],

diff --git a/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml b/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
@@ -56,7 +56,12 @@ spec:
       sourceLabels:
       - __name__
     - action: drop
-      regex: apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)
+      regex: (apiserver_request|apiserver_request_sli|etcd_request)_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)
+      sourceLabels:
+      - __name__
+      - le
+    - action: drop
+      regex: apiserver_request_body_size_bytes_bucket;(150000|350000|550000|650000|850000|950000|(1\.15|1\.35|1\.55|1\.65|1\.85|1\.95|2\.15|2\.35|2\.55|2\.65|2\.85|2\.95)e\+06)
       sourceLabels:
       - __name__
       - le