Merge pull request #786 from machadovilaca/refactor-monitoring-metrics

Refactor monitoring metrics

controllers/ssp_controller.go

@@ -51,6 +51,7 @@ import (
 	"kubevirt.io/ssp-operator/internal/controller/predicates"
 	crd_watch "kubevirt.io/ssp-operator/internal/crd-watch"
 	"kubevirt.io/ssp-operator/internal/operands"
+	"kubevirt.io/ssp-operator/pkg/monitoring/metrics/ssp-operator"
 )
 
 const (
@@ -129,7 +130,7 @@ func (r *sspReconciler) setupController(mgr ctrl.Manager) error {
 func (r *sspReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
 	defer func() {
 		if err != nil {
-			common.SSPOperatorReconcileSucceeded.Set(0)
+			metrics.SetSspOperatorReconcileSucceeded(false)
 		}
 	}()
 	reqLogger := r.log.WithValues("ssp", req.NamespacedName)
@@ -222,9 +223,9 @@ func (r *sspReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 	sspRequest.Logger.Info("CR status updated")
 
 	if sspRequest.Instance.Status.Phase == lifecycleapi.PhaseDeployed {
-		common.SSPOperatorReconcileSucceeded.Set(1)
+		metrics.SetSspOperatorReconcileSucceeded(true)
 	} else {
-		common.SSPOperatorReconcileSucceeded.Set(0)
+		metrics.SetSspOperatorReconcileSucceeded(false)
 	}
 
 	return ctrl.Result{}, nil

controllers/vm_controller.go

@@ -11,7 +11,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
-	"kubevirt.io/ssp-operator/pkg/monitoring/metrics"
+	"kubevirt.io/ssp-operator/pkg/monitoring/metrics/ssp-operator"
 )
 
 // +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines,verbs=get;list;watch

internal/common/resource.go

@@ -8,7 +8,6 @@ import (
 	routev1 "github.com/openshift/api/route/v1"
 	libhandler "github.com/operator-framework/operator-lib/handler"
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
-	"github.com/prometheus/client_golang/prometheus"
 	tekton "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
 	apps "k8s.io/api/apps/v1"
 	core "k8s.io/api/core/v1"
@@ -114,13 +113,6 @@ type reconcileBuilder struct {
 
 var _ ReconcileBuilder = &reconcileBuilder{}
 
-var (
-	SSPOperatorReconcileSucceeded = prometheus.NewGauge(prometheus.GaugeOpts{
-		Name: "kubevirt_ssp_operator_reconcile_succeeded",
-		Help: "Set to 1 if the reconcile process of all operands completes with no errors, and to 0 otherwise",
-	})
-)
-
 func (r *reconcileBuilder) NamespacedResource(resource client.Object) ReconcileBuilder {
 	r.resource = resource
 	r.isClusterResource = false

internal/operands/common-templates/reconcile.go

@@ -14,17 +14,10 @@ import (
 
 	"github.com/blang/semver/v4"
 	"github.com/go-logr/logr"
-	"github.com/prometheus/client_golang/prometheus"
 
 	"kubevirt.io/ssp-operator/internal/common"
 	"kubevirt.io/ssp-operator/internal/operands"
-)
-
-var (
-	CommonTemplatesRestored = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "kubevirt_ssp_common_templates_restored_total",
-		Help: "The total number of common templates restored by the operator back to their original state",
-	})
+	metrics "kubevirt.io/ssp-operator/pkg/monitoring/metrics/ssp-operator"
 )
 
 // Define RBAC rules needed by this operand:
@@ -109,7 +102,7 @@ func incrementTemplatesRestoredMetric(reconcileResults []common.ReconcileResult,
 
 			if reconcileResult.OperationResult == common.OperationResultUpdated && oldVersion == newVersion {
 				logger.Info(fmt.Sprintf("Changes reverted in common template: %s", reconcileResult.Resource.GetName()))
-				CommonTemplatesRestored.Inc()
+				metrics.IncCommonTemplatesRestored()
 			}
 		}
 	}

internal/operands/common-templates/reconcile_test.go

@@ -10,8 +10,6 @@ import (
 
 	templatev1 "github.com/openshift/api/template/v1"
 	libhandler "github.com/operator-framework/operator-lib/handler"
-	"github.com/prometheus/client_golang/prometheus"
-	io_prometheus_client "github.com/prometheus/client_model/go"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	lifecycleapi "kubevirt.io/controller-lifecycle-operator-sdk/api"
@@ -24,6 +22,7 @@ import (
 	"kubevirt.io/ssp-operator/internal/common"
 	"kubevirt.io/ssp-operator/internal/operands"
 	. "kubevirt.io/ssp-operator/internal/test-utils"
+	metrics "kubevirt.io/ssp-operator/pkg/monitoring/metrics/ssp-operator"
 )
 
 var log = logf.Log.WithName("common-templates-operand")
@@ -98,8 +97,8 @@ var _ = Describe("Common-Templates operand", func() {
 			ExpectResourceExists(&template, request)
 		}
 
-		desc, value := getCommonTemplatesRestoredMetric()
-		Expect(desc).To(ContainSubstring("kubevirt_ssp_common_templates_restored_total"))
+		value, err := metrics.GetCommonTemplatesRestored()
+		Expect(err).ToNot(HaveOccurred())
 		Expect(value).To(BeZero())
 	})
 
@@ -131,8 +130,8 @@ var _ = Describe("Common-Templates operand", func() {
 			Expect(template.Labels).To(HaveKey(testLabel))
 		}
 
-		desc, value := getCommonTemplatesRestoredMetric()
-		Expect(desc).To(ContainSubstring("kubevirt_ssp_common_templates_restored_total"))
+		value, err := metrics.GetCommonTemplatesRestored()
+		Expect(err).ToNot(HaveOccurred())
 		Expect(value).To(Equal(float64(len(testTemplates))))
 	})
 
@@ -289,8 +288,8 @@ var _ = Describe("Common-Templates operand", func() {
 			template = getTemplate(request, &testTemplates[0])
 			template.Namespace = namespace
 
-			desc, value := getCommonTemplatesRestoredMetric()
-			Expect(desc).To(ContainSubstring("kubevirt_ssp_common_templates_restored_total"))
+			value, err := metrics.GetCommonTemplatesRestored()
+			Expect(err).ToNot(HaveOccurred())
 			initialMetricValue = value
 		})
 
@@ -305,8 +304,8 @@ var _ = Describe("Common-Templates operand", func() {
 			updatedTpl := getTemplate(request, template)
 			Expect(updatedTpl.Labels[TemplateTypeLabel]).To(Equal(testTemplates[0].Labels[TemplateTypeLabel]))
 
-			desc, value := getCommonTemplatesRestoredMetric()
-			Expect(desc).To(ContainSubstring("kubevirt_ssp_common_templates_restored_total"))
+			value, err := metrics.GetCommonTemplatesRestored()
+			Expect(err).ToNot(HaveOccurred())
 			Expect(value).To(Equal(initialMetricValue + 1))
 		})
 
@@ -322,8 +321,8 @@ var _ = Describe("Common-Templates operand", func() {
 			updatedTpl := getTemplate(request, template)
 			Expect(updatedTpl.Labels[TemplateTypeLabel]).To(Equal(testTemplates[0].Labels[TemplateTypeLabel]))
 
-			desc, value := getCommonTemplatesRestoredMetric()
-			Expect(desc).To(ContainSubstring("kubevirt_ssp_common_templates_restored_total"))
+			value, err := metrics.GetCommonTemplatesRestored()
+			Expect(err).ToNot(HaveOccurred())
 			Expect(value).To(Equal(initialMetricValue))
 		})
 	})
@@ -355,18 +354,6 @@ func getTestTemplates() []templatev1.Template {
 	}}
 }
 
-func getCommonTemplatesRestoredMetric() (string, float64) {
-	ch := make(chan prometheus.Metric, 1)
-	CommonTemplatesRestored.Collect(ch)
-	close(ch)
-	m := <-ch
-	metric := &io_prometheus_client.Metric{}
-	err := m.Write(metric)
-	Expect(err).ToNot(HaveOccurred())
-
-	return m.Desc().String(), metric.GetCounter().GetValue()
-}
-
 func getTemplate(req common.Request, template *templatev1.Template) *templatev1.Template {
 	key := client.ObjectKeyFromObject(template)
 	updatedTpl := &templatev1.Template{}

internal/template-validator/validator/app.go

@@ -19,6 +19,7 @@ import (
 	"kubevirt.io/ssp-operator/internal/template-validator/version"
 	"kubevirt.io/ssp-operator/internal/template-validator/virtinformers"
 	validating "kubevirt.io/ssp-operator/internal/template-validator/webhooks"
+	validatorMetrics "kubevirt.io/ssp-operator/pkg/monitoring/metrics/template-validator"
 )
 
 const (
@@ -75,6 +76,9 @@ func (app *App) Run() {
 
 	registerReadinessProbe()
 
+	// setup monitoring
+	validatorMetrics.SetupMetrics()
+
 	logger.Log.Info("TLS certs directory", "directory", app.TLSInfo.CertsDirectory)
 
 	http.Handle("/metrics", promhttp.Handler())

internal/template-validator/webhooks/hook.go

@@ -24,8 +24,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	admissionv1 "k8s.io/api/admission/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -34,13 +32,7 @@ import (
 	"kubevirt.io/ssp-operator/internal/template-validator/labels"
 	"kubevirt.io/ssp-operator/internal/template-validator/logger"
 	"kubevirt.io/ssp-operator/internal/template-validator/virtinformers"
-)
-
-var (
-	templateValidatorRejected = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "kubevirt_ssp_template_validator_rejected_total",
-		Help: "The total number of rejected template validators",
-	})
+	"kubevirt.io/ssp-operator/pkg/monitoring/metrics/template-validator"
 )
 
 const (
@@ -102,7 +94,7 @@ func (w *webhooks) admitVm(ar *admissionv1.AdmissionReview) *admissionv1.Admissi
 
 	causes := ValidateVm(rules, vm)
 	if len(causes) > 0 {
-		templateValidatorRejected.Inc()
+		metrics.IncTemplateValidatorRejected()
 		return ToAdmissionResponse(causes)
 	}
 

main.go

@@ -44,8 +44,7 @@ import (
 	ssp "kubevirt.io/ssp-operator/api/v1beta2"
 	"kubevirt.io/ssp-operator/controllers"
 	"kubevirt.io/ssp-operator/internal/common"
-	common_templates "kubevirt.io/ssp-operator/internal/operands/common-templates"
-	sspMetrics "kubevirt.io/ssp-operator/pkg/monitoring/metrics"
+	sspMetrics "kubevirt.io/ssp-operator/pkg/monitoring/metrics/ssp-operator"
 	"kubevirt.io/ssp-operator/webhooks"
 	// +kubebuilder:scaffold:imports
 )
@@ -183,6 +182,8 @@ func (s *prometheusServer) getPrometheusTLSConfig(ctx context.Context, certWatch
 }
 
 func newPrometheusServer(metricsAddr string, cache cache.Cache) *prometheusServer {
+	sspMetrics.SetupMetrics()
+
 	return &prometheusServer{
 		certPath:      path.Join(sdkTLSDir, sdkTLSCrt),
 		keyPath:       path.Join(sdkTLSDir, sdkTLSKey),
@@ -203,9 +204,6 @@ func main() {
 	opts := zap.Options{}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
-	metrics.Registry.MustRegister(common_templates.CommonTemplatesRestored)
-	metrics.Registry.MustRegister(common.SSPOperatorReconcileSucceeded)
-	sspMetrics.SetupMetrics()
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 

pkg/monitoring/metrics/metrics.go → pkg/monitoring/metrics/ssp-operator/metrics.go

@@ -5,22 +5,14 @@ import (
 	runtimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
-const metricPrefix = "kubevirt_ssp_"
-
-var (
-	metrics = [][]operatormetrics.Metric{
-		rbdMetrics,
-	}
-)
-
 func SetupMetrics() {
 	operatormetrics.Register = runtimemetrics.Registry.Register
 
-	if err := operatormetrics.RegisterMetrics(metrics...); err != nil {
+	if err := operatormetrics.RegisterMetrics(
+		operatorMetrics,
+		rbdMetrics,
+		templateMetrics,
+	); err != nil {
 		panic(err)
 	}
 }
-
-func ListMetrics() []operatormetrics.Metric {
-	return operatormetrics.ListMetrics()
-}

pkg/monitoring/metrics/ssp-operator/operator_metrics.go

@@ -0,0 +1,24 @@
+package metrics
+
+import "github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+
+var (
+	operatorMetrics = []operatormetrics.Metric{
+		sspOperatorReconcileSucceeded,
+	}
+
+	sspOperatorReconcileSucceeded = operatormetrics.NewGauge(
+		operatormetrics.MetricOpts{
+			Name: "kubevirt_ssp_operator_reconcile_succeeded",
+			Help: "Set to 1 if the reconcile process of all operands completes with no errors, and to 0 otherwise",
+		},
+	)
+)
+
+func SetSspOperatorReconcileSucceeded(isSucceeded bool) {
+	value := 0.0
+	if isSucceeded {
+		value = 1.0
+	}
+	sspOperatorReconcileSucceeded.Set(value)
+}

pkg/monitoring/metrics/rbd_metrics.go → pkg/monitoring/metrics/ssp-operator/rbd_metrics.go

@@ -15,7 +15,7 @@ var (
 
 	vmRbdVolume = operatormetrics.NewGaugeVec(
 		operatormetrics.MetricOpts{
-			Name: metricPrefix + "vm_rbd_block_volume_without_rxbounce",
+			Name: "kubevirt_ssp_vm_rbd_block_volume_without_rxbounce",
 			Help: "VM with RBD mounted Block volume (without rxbounce option set)",
 			ExtraFields: map[string]string{
 				"StabilityLevel": "ALPHA",

pkg/monitoring/metrics/ssp-operator/template_metrics.go

@@ -0,0 +1,32 @@
+package metrics
+
+import (
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+	ioprometheusclient "github.com/prometheus/client_model/go"
+)
+
+var (
+	templateMetrics = []operatormetrics.Metric{
+		commonTemplatesRestored,
+	}
+
+	commonTemplatesRestored = operatormetrics.NewCounter(
+		operatormetrics.MetricOpts{
+			Name: "kubevirt_ssp_common_templates_restored_total",
+			Help: "The total number of common templates restored by the operator back to their original state",
+		},
+	)
+)
+
+func IncCommonTemplatesRestored() {
+	commonTemplatesRestored.Inc()
+}
+
+func GetCommonTemplatesRestored() (float64, error) {
+	dto := &ioprometheusclient.Metric{}
+	err := commonTemplatesRestored.Write(dto)
+	if err != nil {
+		return 0, err
+	}
+	return dto.Counter.GetValue(), nil
+}

pkg/monitoring/metrics/template-validator/metrics.go

@@ -0,0 +1,13 @@
+package metrics
+
+import (
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+)
+
+func SetupMetrics() {
+	if err := operatormetrics.RegisterMetrics(
+		templateMetrics,
+	); err != nil {
+		panic(err)
+	}
+}

pkg/monitoring/metrics/template-validator/template_metrics.go

@@ -0,0 +1,22 @@
+package metrics
+
+import (
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+)
+
+var (
+	templateMetrics = []operatormetrics.Metric{
+		templateValidatorRejected,
+	}
+
+	templateValidatorRejected = operatormetrics.NewCounter(
+		operatormetrics.MetricOpts{
+			Name: "kubevirt_ssp_template_validator_rejected_total",
+			Help: "The total number of rejected template validators",
+		},
+	)
+)
+
+func IncTemplateValidatorRejected() {
+	templateValidatorRejected.Inc()
+}