Merge pull request #143 from robusta-dev/prometheus-metric-fix

Aggregate in case of node change
robusta-dev · Sep 5, 2023 · 5fb3587 · 5fb3587
2 parents 6c64218 + fc7f4ee
commit 5fb3587
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 35 deletions.
diff --git a/robusta_krr/core/integrations/prometheus/metrics/cpu.py b/robusta_krr/core/integrations/prometheus/metrics/cpu.py
@@ -14,14 +14,16 @@ def get_query(self, object: K8sObjectData, duration: str, step: str) -> str:
         pods_selector = "|".join(pod.name for pod in object.pods)
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
-            rate(
-                container_cpu_usage_seconds_total{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{step}]
-            )
+            max(
+                rate(
+                    container_cpu_usage_seconds_total{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}[{step}]
+                )
+            ) by (container, pod, job)
         """
 
 
@@ -40,14 +42,17 @@ def get_query(self, object: K8sObjectData, duration: str, step: str) -> str:
             return f"""
                 quantile_over_time(
                     {round(percentile / 100, 2)},
-                    rate(
-                        container_cpu_usage_seconds_total{{
-                            namespace="{object.namespace}",
-                            pod=~"{pods_selector}",
-                            container="{object.container}"
-                            {cluster_label}
-                        }}[{step}]
-                    )[{duration}:{step}]
+                    max(
+                        rate(
+                            container_cpu_usage_seconds_total{{
+                                namespace="{object.namespace}",
+                                pod=~"{pods_selector}",
+                                container="{object.container}"
+                                {cluster_label}
+                            }}[{step}]
+                        )
+                    ) by (container, pod, job)
+                    [{duration}:{step}]
                 )
             """
 
@@ -64,11 +69,14 @@ def get_query(self, object: K8sObjectData, duration: str, step: str) -> str:
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
             count_over_time(
-                container_cpu_usage_seconds_total{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{duration}]
+                max(
+                    container_cpu_usage_seconds_total{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}
+                ) by (container, pod, job)
+                [{duration}:{step}]
             )
         """
diff --git a/robusta_krr/core/integrations/prometheus/metrics/memory.py b/robusta_krr/core/integrations/prometheus/metrics/memory.py
@@ -14,7 +14,7 @@ def get_query(self, object: K8sObjectData, duration: str, step: str) -> str:
         pods_selector = "|".join(pod.name for pod in object.pods)
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
-            sum(
+            max(
                 container_memory_working_set_bytes{{
                     namespace="{object.namespace}",
                     pod=~"{pods_selector}",
@@ -35,12 +35,15 @@ def get_query(self, object: K8sObjectData, duration: str, step: str) -> str:
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
             max_over_time(
-                container_memory_working_set_bytes{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{duration}:{step}]
+                max(
+                    container_memory_working_set_bytes{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}
+                ) by (container, pod, job)
+                [{duration}:{step}]
             )
         """
 
@@ -55,11 +58,14 @@ def get_query(self, object: K8sObjectData, duration: str, step: str) -> str:
         cluster_label = self.get_prometheus_cluster_label()
         return f"""
             count_over_time(
-                container_memory_working_set_bytes{{
-                    namespace="{object.namespace}",
-                    pod=~"{pods_selector}",
-                    container="{object.container}"
-                    {cluster_label}
-                }}[{duration}:{step}]
+                max(
+                    container_memory_working_set_bytes{{
+                        namespace="{object.namespace}",
+                        pod=~"{pods_selector}",
+                        container="{object.container}"
+                        {cluster_label}
+                    }}
+                ) by (container, pod, job)
+                [{duration}:{step}]
             )
         """