Consolidate metrics new function

Signed-off-by: Jing Liu <[email protected]>

Author: 7ing

examples/simple/deploy/01_simple-csi-driver.yaml

@@ -59,7 +59,7 @@ spec:
             allowPrivilegeEscalation: false
             capabilities: { drop: [ "ALL" ] }
             readOnlyRootFilesystem: true
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.14.0
           args:
             - --v=5
             - --csi-address=/plugin/csi.sock
@@ -94,6 +94,10 @@ spec:
                   fieldPath: spec.nodeName
             - name: CSI_ENDPOINT
               value: unix://plugin/csi.sock
+          ports:
+            - containerPort: 9402
+              name: http-metrics
+              protocol: TCP
           volumeMounts:
             - name: plugin-dir
               mountPath: /plugin

examples/simple/deploy/02_example-app.yaml

@@ -48,7 +48,7 @@ spec:
         runAsUser: 2000
       containers:
         - name: my-frontend
-          image: busybox:1.35.0
+          image: busybox:1.36.1
           volumeMounts:
           - mountPath: "/tls"
             name: tls

examples/simple/main.go

@@ -43,6 +43,7 @@ import (
 	"github.com/cert-manager/csi-lib/metadata"
 	"github.com/cert-manager/csi-lib/metrics"
 	"github.com/cert-manager/csi-lib/storage"
+	"github.com/go-logr/logr"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/sync/errgroup"
 	"k8s.io/client-go/rest"
@@ -113,10 +114,16 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	metricsHandler, err := startMetricsServer(ctx, *nodeID, log, cmClient, store)
-	if err != nil {
-		panic("failed to setup metrics server: " + err.Error())
-	}
+	certRequestInformerFactory := externalversions.NewSharedInformerFactory(cmClient, 5*time.Second)
+	certRequestInformer := certRequestInformerFactory.Certmanager().V1().CertificateRequests()
+	metricsHandler := metrics.New(*nodeID, &log, prometheus.NewRegistry(), store, certRequestInformer.Lister())
+
+	go func() {
+		err := startMetricsServer(ctx, log, metricsHandler, certRequestInformerFactory)
+		if err != nil {
+			panic("failed to setup metrics server: " + err.Error())
+		}
+	}()
 
 	d, err := driver.New(ctx, *endpoint, log, driver.Options{
 		DriverName:    "csi.cert-manager.io",
@@ -367,32 +374,20 @@ func keyUsagesFromAttributes(usagesCSV string) []cmapi.KeyUsage {
 	return keyUsages
 }
 
-// startMetricsServer starts a server listening on port 6443, until the supplied context is cancelled,
+// startMetricsServer starts a server listening on port 9402, until the supplied context is cancelled,
 // after which the server will gracefully shutdown (within 5 seconds).
 func startMetricsServer(
 	rootCtx context.Context,
-	nodeId string,
 	logger logr.Logger,
-	cmClient *cmclient.Clientset,
-	metadataReader storage.MetadataReader,
-) (*metrics.Metrics, error) {
+	metricsHandler *metrics.Metrics,
+	certRequestInformerFactory externalversions.SharedInformerFactory,
+) error {
 	g, ctx := errgroup.WithContext(rootCtx)
-	defer func() {
-		if err := g.Wait(); err != nil {
-			logger.Error(err, "fail to stop metric server")
-		}
-	}()
-
-	metricsHandler := metrics.New(&logger, prometheus.NewRegistry())
-
-	certRequestInformerFactory := externalversions.NewSharedInformerFactory(cmClient, 5*time.Second)
-	certRequestInformer := certRequestInformerFactory.Certmanager().V1().CertificateRequests()
-	metricsHandler.SetupCertificateRequestCollector(nodeId, metadataReader, certRequestInformer.Lister())
 
 	listenConfig := &net.ListenConfig{}
-	metricsLn, err := listenConfig.Listen(ctx, "tcp", "127.0.0.1:6443")
+	metricsLn, err := listenConfig.Listen(ctx, "tcp", ":9402")
 	if err != nil {
-		return nil, err
+		return err
 	}
 	metricsServer := &http.Server{
 		Addr:           metricsLn.Addr().String(),
@@ -423,5 +418,5 @@ func startMetricsServer(
 		}
 		return nil
 	})
-	return metricsHandler, nil
+	return g.Wait()
 }

metrics/certificaterequest_test.go

@@ -215,7 +215,6 @@ func TestCertificateRequestMetrics(t *testing.T) {
 	for n, test := range tests {
 		t.Run(n, func(t *testing.T) {
 			testLog := testr.New(t)
-			m := New(&testLog, prometheus.NewRegistry())
 
 			fakeClient := fake.NewSimpleClientset()
 			factory := externalversions.NewSharedInformerFactory(fakeClient, 0)
@@ -228,7 +227,7 @@ func TestCertificateRequestMetrics(t *testing.T) {
 			assert.NoError(t, err)
 			fakeMetadata := storage.NewMemoryFS()
 			fakeMetadata.RegisterMetadata(test.meta)
-			m.SetupCertificateRequestCollector(testNodeName, fakeMetadata, certRequestInformer.Lister())
+			m := New(testNodeName, &testLog, prometheus.NewRegistry(), fakeMetadata, certRequestInformer.Lister())
 
 			if err := testutil.CollectAndCompare(m.certificateRequestCollector,
 				strings.NewReader(expiryMetadata+test.expectedExpiry),
@@ -354,8 +353,7 @@ func TestCertificateRequestCache(t *testing.T) {
 	assert.NoError(t, err)
 
 	testLog := testr.New(t)
-	m := New(&testLog, prometheus.NewRegistry())
-	m.SetupCertificateRequestCollector(testNodeName, fakeMetadata, certRequestInformer.Lister())
+	m := New(testNodeName, &testLog, prometheus.NewRegistry(), fakeMetadata, certRequestInformer.Lister())
 
 	// Check all three metrics exist
 	if err := testutil.CollectAndCompare(m.certificateRequestCollector,

metrics/metrics.go

@@ -45,7 +45,13 @@ type Metrics struct {
 }
 
 // New creates a Metrics struct and populates it with prometheus metric types.
-func New(logger *logr.Logger, registry *prometheus.Registry) *Metrics {
+func New(
+	nodeId string,
+	logger *logr.Logger,
+	registry *prometheus.Registry,
+	metadataReader storage.MetadataReader,
+	certificateRequestLister cmlisters.CertificateRequestLister,
+) *Metrics {
 	var (
 		// driverIssueCallCountTotal is a Prometheus counter for the number of issue() calls made by the driver.
 		driverIssueCallCountTotal = prometheus.NewCounterVec(
@@ -71,20 +77,26 @@ func New(logger *logr.Logger, registry *prometheus.Registry) *Metrics {
 		)
 	)
 
-	registry.MustRegister(
-		driverIssueCallCountTotal,
-		driverIssueErrorCountTotal,
-	)
-
 	// Create server and register Prometheus metrics handler
 	m := &Metrics{
 		log:      logger.WithName("metrics"),
 		registry: registry,
 
 		driverIssueCallCountTotal:  driverIssueCallCountTotal,
 		driverIssueErrorCountTotal: driverIssueErrorCountTotal,
+		certificateRequestCollector: NewCertificateRequestCollector(
+			internalapiutil.HashIdentifier(nodeId),
+			metadataReader,
+			certificateRequestLister,
+		),
 	}
 
+	m.registry.MustRegister(
+		driverIssueCallCountTotal,
+		driverIssueErrorCountTotal,
+		m.certificateRequestCollector,
+	)
+
 	return m
 }
 

test/integration/metrics_test.go

@@ -100,14 +100,13 @@ func TestMetricsServer(t *testing.T) {
 	testNodeId := "test-node"
 
 	// Build metrics handler, and start metrics server with a random available port
-	metricsHandler := metrics.New(&testLog, prometheus.NewRegistry())
 	store := storage.NewMemoryFS()
 	fakeClient := fake.NewSimpleClientset()
 	// client-go imposes a minimum resync period of 1 second, so that is the lowest we can go
 	// https://github.com/kubernetes/client-go/blob/5a019202120ab4dd7dfb3788e5cb87269f343ebe/tools/cache/shared_informer.go#L575
 	factory := externalversions.NewSharedInformerFactory(fakeClient, time.Second)
 	certRequestInformer := factory.Certmanager().V1().CertificateRequests()
-	metricsHandler.SetupCertificateRequestCollector(testNodeId, store, certRequestInformer.Lister())
+	metricsHandler := metrics.New(testNodeId, &testLog, prometheus.NewRegistry(), store, certRequestInformer.Lister())
 	factory.Start(ctx.Done())
 	factory.WaitForCacheSync(ctx.Done())