Merge pull request #14 from run-ai/new-metrics

New metrics

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 0.0.1
+VERSION ?= 0.0.2
 LOGLEVEL ?= debug
 
 # CHANNELS define the bundle channels used in the bundle.
@@ -159,7 +159,7 @@ docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG}
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
-# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.2). To use this option you need to:
 # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
 # - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 # - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)

README.md

@@ -51,7 +51,7 @@ make install
 ```
 or
 ```sh
-kubectl apply -f github.com/run-ai/argo-rollout-config-keeper/rleasses/v0.0.1/argo-rollout-config-keeper.yaml
+kubectl apply -f github.com/run-ai/argo-rollout-config-keeper/rleasses/v0.0.2/argo-rollout-config-keeper.yaml
 ``` 
 
 

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: docker.io/runaidevops/argo-rollout-config-keeper-operator
-  newTag: v0.0.1
+  newTag: v0.0.2

internal/common/common.go

@@ -76,18 +76,36 @@ func (r *ArgoRolloutConfigKeeperCommon) ReconcileConfigMaps(ctx context.Context,
 					}
 
 					if err := r.finalizerOperation(ctx, &c, finalizerFullName); err != nil {
-						return err
+						r.Logger.Error(err, "unable to remove finalizer from configmap")
+						if namespace != "" {
+							metrics.FailuresInConfigMapClusterScopeCount.Inc()
+						} else {
+							metrics.FailuresInConfigMapCount.Inc()
+						}
+						continue
 					}
 
 					latestVersion, err := r.getLatestVersionOfConfig(ctx, strings.Split(finalizerFullName, "/")[1], &c)
 					if err != nil {
 						r.Logger.Error(err, "unable to get latest version of configmap")
-						return err
+
+						if namespace != "" {
+							metrics.FailuresInConfigMapClusterScopeCount.Inc()
+						} else {
+							metrics.FailuresInConfigMapCount.Inc()
+						}
+						continue
 					}
 
 					err = r.ignoreExtraneousOperation(ctx, &c, latestVersion)
 					if err != nil {
-						return err
+						r.Logger.Error(err, "unable to add IgnoreExtraneous annotation to configmap")
+						if namespace != "" {
+							metrics.FailuresInConfigMapClusterScopeCount.Inc()
+						} else {
+							metrics.FailuresInConfigMapCount.Inc()
+						}
+						continue
 					}
 				} else {
 					r.Logger.Info(fmt.Sprintf("skipping %s configmap, reason: no manageable finalizer", c.Name))
@@ -150,17 +168,34 @@ func (r *ArgoRolloutConfigKeeperCommon) ReconcileSecrets(ctx context.Context, na
 					}
 
 					if err := r.finalizerOperation(ctx, &s, finalizerFullName); err != nil {
+						r.Logger.Error(err, "unable to remove finalizer from secret")
+						if namespace != "" {
+							metrics.FailuresInSecretClusterScopeCount.Inc()
+						} else {
+							metrics.FailuresInSecretCount.Inc()
+						}
 						return err
 					}
 
 					latestVersion, err := r.getLatestVersionOfConfig(ctx, strings.Split(finalizerFullName, "/")[1], &s)
 					if err != nil {
 						r.Logger.Error(err, "unable to get latest version of secret")
-						return err
+						if namespace != "" {
+							metrics.FailuresInSecretClusterScopeCount.Inc()
+						} else {
+							metrics.FailuresInSecretCount.Inc()
+						}
+						continue
 					}
 					err = r.ignoreExtraneousOperation(ctx, &s, latestVersion)
 					if err != nil {
-						return err
+						r.Logger.Error(err, "unable to add IgnoreExtraneous annotation to secret")
+						if namespace != "" {
+							metrics.FailuresInSecretClusterScopeCount.Inc()
+						} else {
+							metrics.FailuresInSecretCount.Inc()
+						}
+						continue
 					}
 				} else {
 					r.Logger.Info(fmt.Sprintf("skipping %s secret, reason: no manageable finalizer", s.Name))

internal/metrics/metrics.go

@@ -26,6 +26,16 @@ var (
 			Name: "argo_rollout_config_keeper_discovered_secret_count",
 			Help: "Number of discovered secrets by argo rollout config keeper operator",
 		})
+	FailuresInConfigMapCount = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "argo_rollout_config_keeper_failures_configmap_count",
+			Help: "Number of failures in configmap reconciliation",
+		})
+	FailuresInSecretCount = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "argo_rollout_config_keeper_failures_secret_count",
+			Help: "Number of failures in secret reconciliation",
+		})
 	ConfigMapReconcileDuration = prometheus.NewHistogram(
 		prometheus.HistogramOpts{
 			Name: "argo_rollout_config_keeper_configmap_reconcile_duration_seconds",
@@ -61,6 +71,16 @@ var (
 			Name: "argo_rollout_config_keeper_discovered_secret_clusterscope_count",
 			Help: "Number of discovered secrets by argo rollout config keeper cluster scope operator",
 		})
+	FailuresInConfigMapClusterScopeCount = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "argo_rollout_config_keeper_failures_configmap_clusterscope_count",
+			Help: "Number of failures in configmap reconciliation by argo rollout config keeper cluster scope operator",
+		})
+	FailuresInSecretClusterScopeCount = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "argo_rollout_config_keeper_failures_secret_clusterscope_count",
+			Help: "Number of failures in secret reconciliation by argo rollout config keeper cluster scope operator",
+		})
 	ConfigMapClusterScopeReconcileDuration = prometheus.NewHistogram(
 		prometheus.HistogramOpts{
 			Name: "argo_rollout_config_keeper_configmap_clusterscope_reconcile_duration_seconds",
@@ -98,13 +118,17 @@ func registerOwnMetrics() []interface{} {
 		ManagedSecretCount,
 		DiscoveredConfigMapCount,
 		DiscoveredSecretCount,
+		FailuresInConfigMapCount,
+		FailuresInSecretCount,
 		ConfigMapReconcileDuration,
 		SecretReconcileDuration,
 		OverallReconcileDuration,
 		ManagedConfigMapClusterScopeCount,
 		ManagedSecretClusterScopeCount,
 		DiscoveredConfigMapClusterScopeCount,
 		DiscoveredSecretClusterScopeCount,
+		FailuresInConfigMapClusterScopeCount,
+		FailuresInSecretClusterScopeCount,
 		ConfigMapClusterScopeReconcileDuration,
 		SecretClusterScopeReconcileDuration,
 		OverallClusterScopeReconcileDuration,

test/e2e/e2e_test.go

@@ -60,7 +60,7 @@ var _ = Describe("controller", Ordered, func() {
 			var err error
 
 			// projectimage stores the name of the image used in the example
-			var projectimage = "example.com/argo-rollout-config-keeper:v0.0.1"
+			var projectimage = "example.com/argo-rollout-config-keeper:v0.0.2"
 
 			By("building the manager(Operator) image")
 			cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))