CHAOSPLT-447: Add a spec field for pausing DisruptionCrons (#933)

* CHAOSPLT-447: Add a spec field for pausing DisruptionCrons

* Add a new metric for counting paused disruptioncrons

Author: ptnapoleon

api/v1beta1/disruption_cron_types.go

@@ -47,6 +47,11 @@ type DisruptionCronSpec struct {
 	// The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron.
 	Schedule string `json:"schedule"`
 
+	// If set to true, no disruptions will be created from this DisruptionCron
+	// useful if there's a reason to temporarily stop injecting, but without
+	// deleting this DisruptionCron
+	Paused bool `json:"paused,omitempty"`
+
 	// Optional deadline for starting the disruption if it misses scheduled time
 	// for any reason.  Missed disruption executions will be counted as failed ones.
 	// +nullable

chart/templates/generated/chaos.datadoghq.com_disruptioncrons.yaml

@@ -630,6 +630,12 @@ spec:
                   required:
                     - count
                   type: object
+                paused:
+                  description: |-
+                    If set to true, no disruptions will be created from this DisruptionCron
+                    useful if there's a reason to temporarily stop injecting, but without
+                    deleting this DisruptionCron
+                  type: boolean
                 reporting:
                   description: |-
                     Reporting provides additional reporting options in order to send a message to a custom slack channel

controllers/disruption_cron_controller.go

@@ -120,6 +120,13 @@ func (r *DisruptionCronReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return ctrl.Result{}, err
 	}
 
+	if instance.Spec.Paused {
+		r.handleMetricSinkError(r.MetricsSink.MetricPausedCron(DisruptionCronTags))
+		r.log.Debugw("disruptioncron has been paused, will not resume until spec.paused is false")
+
+		return ctrl.Result{}, nil
+	}
+
 	// Get next scheduled run or time of unprocessed run
 	// If multiple unmet start times exist, start the last one
 	missedRun, nextRun, err := r.getNextSchedule(instance, time.Now())

docs/disruption_cron.md

@@ -19,6 +19,7 @@ metadata:
   namespace: chaos-demo # it must be in the same namespace as targeted resources
 spec:
   schedule: "*/15 * * * *" # cron syntax specifying that disruption occurs every 15 minutes
+  paused: false # set to true if you want to disable disruption creation temporarily (until set to false again)
   targetResource: # a resource to target
     kind: deployment # a resource name to target
     name: demo-curl # can be either deployment or statefulset

o11y/metrics/datadog/datadog.go

@@ -252,6 +252,11 @@ func (d Sink) MetricDisruptionScheduled(tags []string) error {
 	return d.metricWithStatus(d.prefix+"schedule.disruption_scheduled", tags)
 }
 
+// MetricPausedCron reports when a disruption cron has reconciled in a paused state
+func (d Sink) MetricPausedCron(tags []string) error {
+	return d.metricWithStatus(d.prefix+"schedule.paused", tags)
+}
+
 func boolToStatus(succeed bool) string {
 	var status string
 	if succeed {

o11y/metrics/metrics.go

@@ -51,6 +51,7 @@ type Sink interface {
 	MetricMissingTargetFound(tags []string) error
 	MetricNextScheduledTime(time time.Duration, tags []string) error
 	MetricDisruptionScheduled(tags []string) error
+	MetricPausedCron(tags []string) error
 }
 
 // GetSink returns an initiated sink

o11y/metrics/noop/noop.go

@@ -247,3 +247,10 @@ func (n Sink) MetricDisruptionScheduled(tags []string) error {
 
 	return nil
 }
+
+// MetricPausedCron reports when a disruption cron has reconciled in a paused state
+func (n Sink) MetricPausedCron(tags []string) error {
+	n.log.Debugf("NOOP: MetricPausedCron %s\n", tags)
+
+	return nil
+}