Implement context-based logger propagation (#994)

* feat(o11y): standardize logging with Datadog tag conventions

Centralized all logging tag keys to use constants from o11y/tags/tags.go,
ensuring compliance with Datadog naming conventions (lowercase, underscores).
Replaced hardcoded strings across 64 files with typed constants for
better maintainability and consistency.

Key improvements:
- Organized tag constants by functional clusters (generic, events, timing, etc.)
- Added 80+ missing tag constants for complete coverage
- Removed duplicate constants and mock files
- Enhanced observability infrastructure with structured logging

Co-Authored-By: Claude <[email protected]>

* fix: k8s version

* circle-ci: add auto reruns for e2e-tests

Jira: CHAOSPLT-1211

Author: aymericDD

.circleci/config.yml

@@ -211,6 +211,7 @@ jobs:
             export PATH="/home/circleci/go/bin:${PATH}"
             make e2e-test GOBIN=/home/circleci/go/bin KUBECTL="minikube kubectl --" E2E_TEST_CLUSTER_NAME="minikube" E2E_TEST_KUBECTL_CONTEXT="minikube"
           no_output_timeout: 15m
+          max_auto_reruns: 3
       - run:
           name: Save logs
           when: on_fail

Makefile

@@ -54,7 +54,7 @@ PROTOC_VERSION = 3.17.3
 PROTOC_OS ?= osx
 PROTOC_ZIP = protoc-${PROTOC_VERSION}-${PROTOC_OS}-x86_64.zip
 # you might also want to change ~/lima.yaml k3s version
-KUBERNETES_MAJOR_VERSION ?= 1.26
+KUBERNETES_MAJOR_VERSION ?= 1.28
 KUBERNETES_VERSION ?= v$(KUBERNETES_MAJOR_VERSION).0
 KUBEBUILDER_VERSION ?= 3.1.0
 USE_VOLUMES ?= false
@@ -70,7 +70,7 @@ GOLANGCI_LINT_INSTALLED_VERSION = $(shell (golangci-lint --version || echo "") |
 CONTROLLER_GEN_VERSION = v0.14.0
 CONTROLLER_GEN_INSTALLED_VERSION = $(shell (controller-gen --version || echo "") | awk '{ print $$2 }')
 
-MOCKERY_VERSION = 2.53.0
+MOCKERY_VERSION = 2.53.5
 MOCKERY_ARCH = $(GOARCH)
 ifeq (amd64,$(GOARCH))
 MOCKERY_ARCH = x86_64

api/v1beta1/disruption_cron_types.go

@@ -9,9 +9,9 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/DataDog/chaos-controller/log"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	tagutil "github.com/DataDog/chaos-controller/o11y/tags"
 )
 
 func init() {
@@ -119,8 +119,8 @@ func (d *DisruptionCron) IsReadyToRemoveFinalizer(finalizerDelay time.Duration)
 // getMetricsTags parses the disruptioncron to generate metrics tags
 func (d *DisruptionCron) getMetricsTags() []string {
 	tags := []string{
-		fmt.Sprintf("%s:%s", log.DisruptionCronNameKey, d.Name),
-		fmt.Sprintf("%s:%s", log.DisruptionCronNamespaceKey, d.Namespace),
+		tagutil.FormatTag(tagutil.DisruptionCronNameKey, d.Name),
+		tagutil.FormatTag(tagutil.DisruptionCronNamespaceKey, d.Namespace),
 	}
 
 	return tags

api/v1beta1/disruption_cron_webhook.go

@@ -12,10 +12,6 @@ import (
 	"strings"
 	"time"
 
-	cLog "github.com/DataDog/chaos-controller/log"
-	"github.com/DataDog/chaos-controller/o11y/metrics"
-	"github.com/DataDog/chaos-controller/utils"
-
 	"github.com/robfig/cron"
 	"go.uber.org/zap"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,6 +22,10 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	"github.com/DataDog/chaos-controller/o11y/metrics"
+	tagutil "github.com/DataDog/chaos-controller/o11y/tags"
+	"github.com/DataDog/chaos-controller/utils"
 )
 
 const (
@@ -48,8 +48,8 @@ func (d *DisruptionCron) SetupWebhookWithManager(setupWebhookConfig utils.SetupW
 	disruptionCronWebhookRecorder = setupWebhookConfig.Recorder
 	disruptionCronWebhookDeleteOnly = setupWebhookConfig.DeleteOnlyFlag
 	disruptionCronWebhookLogger = setupWebhookConfig.Logger.With(
-		"source", "admission-controller",
-		"admission-controller", "disruption-cron-webhook",
+		tagutil.SourceKey, "admission-controller",
+		tagutil.AdmissionControllerKey, "disruption-cron-webhook",
 	)
 	disruptionCronMetricsSink = setupWebhookConfig.MetricsSink
 
@@ -76,7 +76,11 @@ var _ webhook.Defaulter = &DisruptionCron{}
 // Default implements webhook.Defaulter so a webhook will be registered for the type
 func (d *DisruptionCron) Default() {
 	if d.Spec.DelayedStartTolerance.Duration() == 0 {
-		logger.Infow(fmt.Sprintf("setting default delayedStartTolerance of %s in disruptionCron", defaultCronDelayedStartTolerance), cLog.DisruptionCronNameKey, d.Name, cLog.DisruptionCronNamespaceKey, d.Namespace)
+		logger.Infow(fmt.Sprintf("setting default delayedStartTolerance of %s in disruptionCron", defaultCronDelayedStartTolerance),
+			tagutil.DisruptionCronNameKey, d.Name,
+			tagutil.DisruptionCronNamespaceKey, d.Namespace,
+		)
+
 		d.Spec.DelayedStartTolerance = DisruptionDuration(defaultCronDelayedStartTolerance.String())
 	}
 }
@@ -87,16 +91,19 @@ var _ webhook.Validator = &DisruptionCron{}
 
 // ValidateCreate implements webhook.Validator so a webhook will be registered for the type
 func (d *DisruptionCron) ValidateCreate() (_ admission.Warnings, err error) {
-	log := disruptionCronWebhookLogger.With(cLog.DisruptionCronNameKey, d.Name, cLog.DisruptionCronNamespaceKey, d.Namespace)
+	log := disruptionCronWebhookLogger.With(
+		tagutil.DisruptionCronNameKey, d.Name,
+		tagutil.DisruptionCronNamespaceKey, d.Namespace,
+	)
 
-	log.Infow("validating created disruption cron", "spec", d.Spec)
+	log.Infow("validating created disruption cron", tagutil.SpecKey, d.Spec)
 
 	metricTags := d.getMetricsTags()
 
 	defer func() {
 		if err != nil {
 			if mErr := disruptionCronMetricsSink.MetricValidationFailed(metricTags); mErr != nil {
-				log.Errorw("error sending a metric", "error", mErr)
+				log.Errorw("error sending a metric", tagutil.ErrorKey, mErr)
 			}
 		}
 	}()
@@ -123,7 +130,7 @@ func (d *DisruptionCron) ValidateCreate() (_ admission.Warnings, err error) {
 	}
 
 	if mErr := metricsSink.MetricValidationCreated(metricTags); mErr != nil {
-		log.Errorw("error sending a metric", "error", mErr)
+		log.Errorw("error sending a metric", tagutil.ErrorKey, mErr)
 	}
 
 	// send informative event to disruption cron to broadcast
@@ -133,16 +140,18 @@ func (d *DisruptionCron) ValidateCreate() (_ admission.Warnings, err error) {
 }
 
 func (d *DisruptionCron) ValidateUpdate(oldObject runtime.Object) (_ admission.Warnings, err error) {
-	log := logger.With(cLog.DisruptionCronNameKey, d.Name, cLog.DisruptionCronNamespaceKey, d.Namespace)
+	log := logger.With(
+		tagutil.DisruptionCronNameKey, d.Name, tagutil.DisruptionCronNamespaceKey, d.Namespace,
+	)
 
-	log.Infow("validating updated disruption cron", "spec", d.Spec)
+	log.Infow("validating updated disruption cron", tagutil.SpecKey, d.Spec)
 
 	metricTags := d.getMetricsTags()
 
 	defer func() {
 		if err != nil {
 			if mErr := disruptionCronMetricsSink.MetricValidationFailed(metricTags); mErr != nil {
-				log.Errorw("error sending a metric", "error", mErr)
+				log.Errorw("error sending a metric", tagutil.ErrorKey, mErr)
 			}
 		}
 	}()
@@ -169,7 +178,7 @@ func (d *DisruptionCron) ValidateUpdate(oldObject runtime.Object) (_ admission.W
 	}
 
 	if mErr := metricsSink.MetricValidationUpdated(metricTags); mErr != nil {
-		log.Errorw("error sending a metric", "error", mErr)
+		log.Errorw("error sending a metric", tagutil.ErrorKey, mErr)
 	}
 
 	// send informative event to disruption cron to broadcast
@@ -179,15 +188,18 @@ func (d *DisruptionCron) ValidateUpdate(oldObject runtime.Object) (_ admission.W
 }
 
 func (d *DisruptionCron) ValidateDelete() (warnings admission.Warnings, err error) {
-	log := disruptionCronWebhookLogger.With(cLog.DisruptionCronNameKey, d.Name, cLog.DisruptionCronNamespaceKey, d.Namespace)
+	log := disruptionCronWebhookLogger.With(
+		tagutil.DisruptionCronNameKey, d.Name,
+		tagutil.DisruptionCronNamespaceKey, d.Namespace,
+	)
 
-	log.Infow("validating deleted disruption cron", "spec", d.Spec)
+	log.Infow("validating deleted disruption cron", tagutil.SpecKey, d.Spec)
 
 	// During the validation of the deletion the timestamp does not exist so we need to set it before emitting the event
 	d.DeletionTimestamp = &metav1.Time{Time: time.Now()}
 
 	if mErr := metricsSink.MetricValidationDeleted(d.getMetricsTags()); mErr != nil {
-		log.Errorw("error sending a metric", "error", mErr)
+		log.Errorw("error sending a metric", tagutil.ErrorKey, mErr)
 	}
 
 	// send informative event to disruption cron to broadcast
@@ -199,7 +211,7 @@ func (d *DisruptionCron) ValidateDelete() (warnings admission.Warnings, err erro
 func (d *DisruptionCron) emitEvent(eventReason EventReason) {
 	disruptionCronJSON, err := json.Marshal(d)
 	if err != nil {
-		disruptionCronWebhookLogger.Errorw("failed to marshal disruption cron", "error", err)
+		disruptionCronWebhookLogger.Errorw("failed to marshal disruption cron", tagutil.ErrorKey, err)
 		return
 	}
 

api/v1beta1/disruption_pods.go

@@ -9,16 +9,18 @@ import (
 	"context"
 	"fmt"
 
-	chaostypes "github.com/DataDog/chaos-controller/types"
-	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	cLog "github.com/DataDog/chaos-controller/log"
+	"github.com/DataDog/chaos-controller/o11y/tags"
+	chaostypes "github.com/DataDog/chaos-controller/types"
 )
 
 // GetChaosPods returns chaos pods owned by the given instance and having the given labels
 // both instance and label set are optional but at least one must be provided
-func GetChaosPods(ctx context.Context, log *zap.SugaredLogger, chaosNamespace string, k8sClient client.Client, instance *Disruption, ls labels.Set) ([]corev1.Pod, error) {
+func GetChaosPods(ctx context.Context, chaosNamespace string, k8sClient client.Client, instance *Disruption, ls labels.Set) ([]corev1.Pod, error) {
 	pods := &corev1.PodList{}
 
 	if k8sClient == nil {
@@ -56,9 +58,11 @@ func GetChaosPods(ctx context.Context, log *zap.SugaredLogger, chaosNamespace st
 		podNames = append(podNames, pod.Name)
 	}
 
-	if log != nil {
-		log.Debugw("searching for chaos pods with label selector...", "labels", ls.String(), "foundPods", podNames)
-	}
+	// Get logger from context for debugging
+	cLog.FromContext(ctx).Debugw("searching for chaos pods with label selector...",
+		tags.LabelsKey, ls.String(),
+		tags.FoundPodsKey, podNames,
+	)
 
 	return pods.Items, nil
 }