Replace stuck on removal metrics with new stuck_on_removal_current (#978)

* Replace existing stuck on removal metrics

* Use new current stuck metric

Author: ptnapoleon

controllers/disruption_controller.go

@@ -24,16 +24,6 @@ import (
 	"strings"
 	"time"
 
-	chaosv1beta1 "github.com/DataDog/chaos-controller/api/v1beta1"
-	"github.com/DataDog/chaos-controller/cloudservice"
-	cLog "github.com/DataDog/chaos-controller/log"
-	"github.com/DataDog/chaos-controller/o11y/metrics"
-	"github.com/DataDog/chaos-controller/o11y/tracer"
-	"github.com/DataDog/chaos-controller/safemode"
-	"github.com/DataDog/chaos-controller/services"
-	"github.com/DataDog/chaos-controller/targetselector"
-	chaostypes "github.com/DataDog/chaos-controller/types"
-	"github.com/DataDog/chaos-controller/watchers"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -56,6 +46,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	chaosv1beta1 "github.com/DataDog/chaos-controller/api/v1beta1"
+	"github.com/DataDog/chaos-controller/cloudservice"
+	cLog "github.com/DataDog/chaos-controller/log"
+	"github.com/DataDog/chaos-controller/o11y/metrics"
+	"github.com/DataDog/chaos-controller/o11y/tracer"
+	"github.com/DataDog/chaos-controller/safemode"
+	"github.com/DataDog/chaos-controller/services"
+	"github.com/DataDog/chaos-controller/targetselector"
+	chaostypes "github.com/DataDog/chaos-controller/types"
+	"github.com/DataDog/chaos-controller/watchers"
 )
 
 // DisruptionReconciler reconciles a Disruption object
@@ -1054,7 +1055,7 @@ func (r *DisruptionReconciler) ReportMetrics(ctx context.Context) {
 			if d.Status.IsStuckOnRemoval {
 				stuckOnRemoval++
 
-				if err := r.MetricsSink.MetricStuckOnRemoval([]string{"disruptionName:" + d.Name, "namespace:" + d.Namespace}); err != nil {
+				if err := r.MetricsSink.MetricStuckOnRemovalCurrent(1, []string{"disruptionName:" + d.Name, "disruptionNamespace:" + d.Namespace}); err != nil {
 					r.BaseLog.Errorw("error sending stuck_on_removal metric", "error", err)
 				}
 			}
@@ -1071,11 +1072,6 @@ func (r *DisruptionReconciler) ReportMetrics(ctx context.Context) {
 			namespaces[d.Namespace]++
 		}
 
-		// send metrics
-		if err := r.MetricsSink.MetricStuckOnRemovalGauge(float64(stuckOnRemoval)); err != nil {
-			r.BaseLog.Errorw("error sending stuck_on_removal_total metric", "error", err)
-		}
-
 		if len(namespaces) > 0 {
 			for namespace, count := range namespaces {
 				if err := r.MetricsSink.MetricDisruptionsGauge(float64(count), []string{fmt.Sprintf("namespace:%s", namespace)}); err != nil {

o11y/metrics/datadog/datadog.go

@@ -10,9 +10,10 @@ import (
 	"os"
 	"time"
 
+	"github.com/DataDog/datadog-go/statsd"
+
 	"github.com/DataDog/chaos-controller/o11y/metrics/types"
 	chaostypes "github.com/DataDog/chaos-controller/types"
-	"github.com/DataDog/datadog-go/statsd"
 )
 
 const (
@@ -163,17 +164,11 @@ func (d Sink) MetricPodsCreated(target, instanceName, namespace string, succeed
 	return d.metricWithStatus(d.prefix+"pods.created", tags)
 }
 
-// MetricStuckOnRemoval is emitted once per minute per disruption, if that disruption is "stuck on removal", i.e.,
+// MetricStuckOnRemovalCurrent is emitted once per minute counting the number of disruptions _per namespace_
+// that are "stuck on removal", i.e.,
 // we have attempted to clean and delete the disruption, but that has not worked, and a human needs to intervene.
-func (d Sink) MetricStuckOnRemoval(tags []string) error {
-	return d.metricWithStatus(d.prefix+"disruptions.stuck_on_removal", tags)
-}
-
-// MetricStuckOnRemovalGauge is emitted once per minute counting the total number of disruptions that are
-// "stuck on removal", i.e., we have attempted to clean and delete the disruption, but that has not worked,
-// and a human needs to intervene.
-func (d Sink) MetricStuckOnRemovalGauge(gauge float64) error {
-	return d.client.Gauge(d.prefix+"disruptions.stuck_on_removal_total", gauge, []string{}, 1)
+func (d Sink) MetricStuckOnRemovalCurrent(gauge float64, tags []string) error {
+	return d.client.Gauge(d.prefix+"disruptions.stuck_on_removal_current", gauge, tags, 1)
 }
 
 // MetricDisruptionsGauge is emitted once per minute counting the total number of ongoing disruptions per namespace,

o11y/metrics/metrics.go

@@ -9,11 +9,12 @@ import (
 	"fmt"
 	"time"
 
+	"go.uber.org/zap"
+
 	"github.com/DataDog/chaos-controller/o11y/metrics/datadog"
 	"github.com/DataDog/chaos-controller/o11y/metrics/noop"
 	"github.com/DataDog/chaos-controller/o11y/metrics/types"
 	chaostypes "github.com/DataDog/chaos-controller/types"
-	"go.uber.org/zap"
 )
 
 // Sink describes a metric sink
@@ -56,13 +57,10 @@ type Sink interface {
 	// MetricDisruptionOngoingDuration indicates the duration between a Disruption's creation timestamp, and the current time.
 	// This is emitted approximately every one minute
 	MetricDisruptionOngoingDuration(duration time.Duration, tags []string) error
-	// MetricStuckOnRemoval is emitted once per minute per disruption, if that disruption is "stuck on removal", i.e.,
+	// MetricStuckOnRemovalCurrent is emitted once per minute counting the number of disruptions _per namespace_
+	// that are "stuck on removal", i.e.,
 	// we have attempted to clean and delete the disruption, but that has not worked, and a human needs to intervene.
-	MetricStuckOnRemoval(tags []string) error
-	// MetricStuckOnRemovalGauge is emitted once per minute counting the total number of disruptions that are
-	// "stuck on removal", i.e., we have attempted to clean and delete the disruption, but that has not worked,
-	// and a human needs to intervene.
-	MetricStuckOnRemovalGauge(gauge float64) error
+	MetricStuckOnRemovalCurrent(gauge float64, tags []string) error
 	// MetricDisruptionsGauge is emitted once per minute counting the total number of ongoing disruptions per namespace,
 	// or if we fail to determine the namespaced metrics, simply the total number of disruptions found
 	MetricDisruptionsGauge(gauge float64, tags []string) error

o11y/metrics/noop/noop.go

@@ -9,9 +9,10 @@ import (
 	"fmt"
 	"time"
 
+	"go.uber.org/zap"
+
 	"github.com/DataDog/chaos-controller/o11y/metrics/types"
 	chaostypes "github.com/DataDog/chaos-controller/types"
-	"go.uber.org/zap"
 )
 
 // Sink describes a no-op sink
@@ -128,19 +129,11 @@ func (n Sink) MetricPodsCreated(target, instanceName, namespace string, succeed
 	return nil
 }
 
-// MetricStuckOnRemoval is emitted once per minute per disruption, if that disruption is "stuck on removal", i.e.,
+// MetricStuckOnRemovalCurrent is emitted once per minute counting the number of disruptions _per namespace_
+// that are "stuck on removal", i.e.,
 // we have attempted to clean and delete the disruption, but that has not worked, and a human needs to intervene.
-func (n Sink) MetricStuckOnRemoval(tags []string) error {
-	fmt.Println("NOOP: MetricStuckOnRemoval +1")
-
-	return nil
-}
-
-// MetricStuckOnRemovalGauge is emitted once per minute counting the total number of disruptions that are
-// "stuck on removal", i.e., we have attempted to clean and delete the disruption, but that has not worked,
-// and a human needs to intervene.
-func (n Sink) MetricStuckOnRemovalGauge(gauge float64) error {
-	n.log.Debugf("NOOP: MetricStuckOnRemovalGauge %f\n", gauge)
+func (n Sink) MetricStuckOnRemovalCurrent(gauge float64, tags []string) error {
+	n.log.Debugf("NOOP: MetricStuckOnRemovalCurrent %f %s\n", gauge, tags)
 
 	return nil
 }