Skip to content

Commit 14d62c6

Browse files
7ing7ing
committed
Add metric example to simple-csi
Signed-off-by: Jing Liu <[email protected]>
1 parent d6048b8 commit 14d62c6

File tree

6 files changed

+89
-10
lines changed

6 files changed

+89
-10
lines changed

‎examples/simple/go.mod‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ replace github.com/cert-manager/csi-lib => ../../
77
require (
88
github.com/cert-manager/cert-manager v1.18.2
99
github.com/cert-manager/csi-lib v0.0.0-00010101000000-000000000000
10+
github.com/go-logr/logr v1.4.3
11+
github.com/prometheus/client_golang v1.22.0
12+
golang.org/x/sync v0.15.0
1013
k8s.io/client-go v0.34.0
1114
k8s.io/klog/v2 v2.130.1
1215
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
@@ -23,7 +26,6 @@ require (
2326
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
2427
github.com/go-asn1-ber/asn1-ber v1.5.6 // indirect
2528
github.com/go-ldap/ldap/v3 v3.4.8 // indirect
26-
github.com/go-logr/logr v1.4.3 // indirect
2729
github.com/go-logr/zapr v1.3.0 // indirect
2830
github.com/go-openapi/jsonpointer v0.21.0 // indirect
2931
github.com/go-openapi/jsonreference v0.21.0 // indirect
@@ -43,7 +45,6 @@ require (
4345
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
4446
github.com/pkg/errors v0.9.1 // indirect
4547
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
46-
github.com/prometheus/client_golang v1.22.0 // indirect
4748
github.com/prometheus/client_model v0.6.1 // indirect
4849
github.com/prometheus/common v0.62.0 // indirect
4950
github.com/prometheus/procfs v0.15.1 // indirect

‎examples/simple/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
198198
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
199199
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
200200
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
201+
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
202+
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
201203
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
202204
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
203205
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

‎examples/simple/main.go‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,21 +28,27 @@ import (
2828
"flag"
2929
"fmt"
3030
"net"
31+
"net/http"
3132
"net/url"
3233
"strings"
3334
"time"
3435

3536
cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
3637
cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
3738
cmclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
39+
"github.com/cert-manager/cert-manager/pkg/client/informers/externalversions"
3840
"github.com/cert-manager/cert-manager/pkg/util/pki"
41+
"github.com/go-logr/logr"
42+
"github.com/prometheus/client_golang/prometheus"
43+
"golang.org/x/sync/errgroup"
3944
"k8s.io/client-go/rest"
4045
"k8s.io/klog/v2/klogr"
4146
"k8s.io/utils/clock"
4247

4348
"github.com/cert-manager/csi-lib/driver"
4449
"github.com/cert-manager/csi-lib/manager"
4550
"github.com/cert-manager/csi-lib/metadata"
51+
"github.com/cert-manager/csi-lib/metrics"
4652
"github.com/cert-manager/csi-lib/storage"
4753
)
4854

@@ -104,13 +110,23 @@ func main() {
104110

105111
store.FSGroupVolumeAttributeKey = FsGroupKey
106112

107-
d, err := driver.New(context.Background(), *endpoint, log, driver.Options{
113+
cmClient := cmclient.NewForConfigOrDie(restConfig)
114+
115+
ctx, cancel := context.WithCancel(context.Background())
116+
defer cancel()
117+
118+
metricsHandler, err := startMetricsServer(ctx, *nodeID, log, cmClient, store)
119+
if err != nil {
120+
panic("failed to setup metrics server: " + err.Error())
121+
}
122+
123+
d, err := driver.New(ctx, *endpoint, log, driver.Options{
108124
DriverName: "csi.cert-manager.io",
109125
DriverVersion: "v0.0.1",
110126
NodeID: *nodeID,
111127
Store: store,
112128
Manager: manager.NewManagerOrDie(manager.Options{
113-
Client: cmclient.NewForConfigOrDie(restConfig),
129+
Client: cmClient,
114130
MetadataReader: store,
115131
Clock: clock.RealClock{},
116132
Log: &log,
@@ -119,6 +135,7 @@ func main() {
119135
GenerateRequest: generateRequest,
120136
SignRequest: signRequest,
121137
WriteKeypair: (&writer{store: store}).writeKeypair,
138+
Metrics: metricsHandler,
122139
}),
123140
})
124141
if err != nil {
@@ -351,3 +368,62 @@ func keyUsagesFromAttributes(usagesCSV string) []cmapi.KeyUsage {
351368

352369
return keyUsages
353370
}
371+
372+
// startMetricsServer starts a server listening on port 6443, until the supplied context is cancelled,
373+
// after which the server will gracefully shutdown (within 5 seconds).
374+
func startMetricsServer(
375+
rootCtx context.Context,
376+
nodeId string,
377+
logger logr.Logger,
378+
cmClient *cmclient.Clientset,
379+
metadataReader storage.MetadataReader,
380+
) (*metrics.Metrics, error) {
381+
g, ctx := errgroup.WithContext(rootCtx)
382+
defer func() {
383+
if err := g.Wait(); err != nil {
384+
logger.Error(err, "fail to stop metric server")
385+
}
386+
}()
387+
388+
metricsHandler := metrics.New(&logger, prometheus.NewRegistry())
389+
390+
certRequestInformerFactory := externalversions.NewSharedInformerFactory(cmClient, 5*time.Second)
391+
certRequestInformer := certRequestInformerFactory.Certmanager().V1().CertificateRequests()
392+
metricsHandler.SetupCertificateRequestCollector(nodeId, metadataReader, certRequestInformer.Lister())
393+
394+
listenConfig := &net.ListenConfig{}
395+
metricsLn, err := listenConfig.Listen(ctx, "tcp", "127.0.0.1:6443")
396+
if err != nil {
397+
return nil, err
398+
}
399+
metricsServer := &http.Server{
400+
Addr: metricsLn.Addr().String(),
401+
ReadTimeout: 8 * time.Second,
402+
WriteTimeout: 8 * time.Second,
403+
MaxHeaderBytes: 1 << 20, // 1 MiB
404+
Handler: metricsHandler.DefaultHandler(),
405+
}
406+
407+
g.Go(func() error {
408+
certRequestInformerFactory.Start(ctx.Done())
409+
certRequestInformerFactory.WaitForCacheSync(ctx.Done())
410+
return nil
411+
})
412+
g.Go(func() error {
413+
<-rootCtx.Done()
414+
// allow a timeout for graceful shutdown
415+
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
416+
defer cancel()
417+
418+
// nolint: contextcheck
419+
return metricsServer.Shutdown(shutdownCtx)
420+
})
421+
g.Go(func() error {
422+
logger.Info("starting metrics server", "address", metricsLn.Addr())
423+
if err := metricsServer.Serve(metricsLn); err != http.ErrServerClosed {
424+
return err
425+
}
426+
return nil
427+
})
428+
return metricsHandler, nil
429+
}

‎metrics/certificaterequest_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ func TestCertificateRequestMetrics(t *testing.T) {
228228
assert.NoError(t, err)
229229
fakeMetadata := storage.NewMemoryFS()
230230
fakeMetadata.RegisterMetadata(test.meta)
231-
m.SetupCertificateRequestCollector(internalapiutil.HashIdentifier(testNodeName), fakeMetadata, certRequestInformer.Lister())
231+
m.SetupCertificateRequestCollector(testNodeName, fakeMetadata, certRequestInformer.Lister())
232232

233233
if err := testutil.CollectAndCompare(m.certificateRequestCollector,
234234
strings.NewReader(expiryMetadata+test.expectedExpiry),
@@ -355,7 +355,7 @@ func TestCertificateRequestCache(t *testing.T) {
355355

356356
testLog := testr.New(t)
357357
m := New(&testLog, prometheus.NewRegistry())
358-
m.SetupCertificateRequestCollector(testNodeNameHash, fakeMetadata, certRequestInformer.Lister())
358+
m.SetupCertificateRequestCollector(testNodeName, fakeMetadata, certRequestInformer.Lister())
359359

360360
// Check all three metrics exist
361361
if err := testutil.CollectAndCompare(m.certificateRequestCollector,

‎metrics/metrics.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/prometheus/client_golang/prometheus"
2525
"github.com/prometheus/client_golang/prometheus/promhttp"
2626

27+
internalapiutil "github.com/cert-manager/csi-lib/internal/api/util"
2728
"github.com/cert-manager/csi-lib/storage"
2829
)
2930

@@ -95,8 +96,8 @@ func (m *Metrics) DefaultHandler() http.Handler {
9596
return mux
9697
}
9798

98-
func (m *Metrics) SetupCertificateRequestCollector(nodeNameHash string, metadataReader storage.MetadataReader, certificateRequestLister cmlisters.CertificateRequestLister) {
99-
m.certificateRequestCollector = NewCertificateRequestCollector(nodeNameHash, metadataReader, certificateRequestLister)
99+
func (m *Metrics) SetupCertificateRequestCollector(nodeId string, metadataReader storage.MetadataReader, certificateRequestLister cmlisters.CertificateRequestLister) {
100+
m.certificateRequestCollector = NewCertificateRequestCollector(internalapiutil.HashIdentifier(nodeId), metadataReader, certificateRequestLister)
100101
m.registry.MustRegister(m.certificateRequestCollector)
101102
}
102103

‎test/integration/metrics_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ import (
4141
"k8s.io/apimachinery/pkg/util/wait"
4242
fakeclock "k8s.io/utils/clock/testing"
4343

44-
internalapiutil "github.com/cert-manager/csi-lib/internal/api/util"
4544
"github.com/cert-manager/csi-lib/manager"
4645
"github.com/cert-manager/csi-lib/metadata"
4746
"github.com/cert-manager/csi-lib/metrics"
@@ -108,7 +107,7 @@ func TestMetricsServer(t *testing.T) {
108107
// https://github.com/kubernetes/client-go/blob/5a019202120ab4dd7dfb3788e5cb87269f343ebe/tools/cache/shared_informer.go#L575
109108
factory := externalversions.NewSharedInformerFactory(fakeClient, time.Second)
110109
certRequestInformer := factory.Certmanager().V1().CertificateRequests()
111-
metricsHandler.SetupCertificateRequestCollector(internalapiutil.HashIdentifier(testNodeId), store, certRequestInformer.Lister())
110+
metricsHandler.SetupCertificateRequestCollector(testNodeId, store, certRequestInformer.Lister())
112111
factory.Start(ctx.Done())
113112
factory.WaitForCacheSync(ctx.Done())
114113

0 commit comments

Comments
 (0)