Skip to content

Commit d196109

Browse files
authored
#3663: add Prometheus metrics to crypto storage engine (#3786)
1 parent e29ccd1 commit d196109

File tree

5 files changed

+286
-57
lines changed

5 files changed

+286
-57
lines changed

crypto/crypto.go

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/nuts-foundation/nuts-node/crypto/storage/azure"
2828
"github.com/nuts-foundation/nuts-node/storage"
2929
"github.com/nuts-foundation/nuts-node/storage/orm"
30+
"github.com/prometheus/client_golang/prometheus"
3031
"gorm.io/gorm"
3132
"path"
3233
"time"
@@ -65,6 +66,7 @@ func DefaultCryptoConfig() Config {
6566
}
6667

6768
var _ KeyStore = (*Crypto)(nil)
69+
var _ core.Runnable = (*Crypto)(nil)
6870

6971
// Crypto holds references to storage and needed config
7072
type Crypto struct {
@@ -74,6 +76,22 @@ type Crypto struct {
7476
storage storage.Engine
7577
}
7678

79+
func (client *Crypto) Start() error {
80+
for _, collector := range client.backend.(*spi.PrometheusWrapper).Collectors() {
81+
if err := prometheus.Register(collector); err != nil && !errors.Is(err, prometheus.AlreadyRegisteredError{}) {
82+
return fmt.Errorf("register metric: %w", err)
83+
}
84+
}
85+
return nil
86+
}
87+
88+
func (client *Crypto) Shutdown() error {
89+
for _, collector := range client.backend.(*spi.PrometheusWrapper).Collectors() {
90+
_ = prometheus.Unregister(collector)
91+
}
92+
return nil
93+
}
94+
7795
func (client *Crypto) CheckHealth() map[string]core.Health {
7896
return client.backend.CheckHealth()
7997
}
@@ -94,49 +112,28 @@ func (client *Crypto) Config() interface{} {
94112
return &client.config
95113
}
96114

97-
func (client *Crypto) setupFSBackend(config core.ServerConfig) error {
115+
func (client *Crypto) setupFSBackend(config core.ServerConfig) (spi.Storage, error) {
98116
log.Logger().Info("Setting up FileSystem backend for storage of private key material. " +
99117
"Discouraged for production use unless backups and encryption is properly set up. Consider using the Hashicorp Vault backend.")
100118
fsPath := path.Join(config.Datadir, "crypto")
101-
fsBackend, err := fs.NewFileSystemBackend(fsPath)
102-
if err != nil {
103-
return err
104-
}
105-
client.backend = spi.NewValidatedKIDBackendWrapper(fsBackend, spi.KidPattern)
106-
return nil
119+
return fs.NewFileSystemBackend(fsPath)
107120
}
108121

109-
func (client *Crypto) setupStorageAPIBackend() error {
122+
func (client *Crypto) setupStorageAPIBackend() (spi.Storage, error) {
110123
log.Logger().Debug("Setting up StorageAPI backend for storage of private key material.")
111124
log.Logger().Warn("External key storage backend is deprecated and will be removed in the future.")
112-
apiBackend, err := external.NewAPIClient(client.config.External)
113-
if err != nil {
114-
return fmt.Errorf("unable to set up external crypto API client: %w", err)
115-
}
116-
client.backend = spi.NewValidatedKIDBackendWrapper(apiBackend, spi.KidPattern)
117-
return nil
125+
return external.NewAPIClient(client.config.External)
118126
}
119127

120-
func (client *Crypto) setupVaultBackend(_ core.ServerConfig) error {
128+
func (client *Crypto) setupVaultBackend(_ core.ServerConfig) (spi.Storage, error) {
121129
log.Logger().Debug("Setting up Vault backend for storage of private key material. " +
122130
"This feature is experimental and may change in the future.")
123-
vaultBackend, err := vault.NewVaultKVStorage(client.config.Vault)
124-
if err != nil {
125-
return err
126-
}
127-
128-
client.backend = spi.NewValidatedKIDBackendWrapper(vaultBackend, spi.KidPattern)
129-
return nil
131+
return vault.NewVaultKVStorage(client.config.Vault)
130132
}
131133

132-
func (client *Crypto) setupAzureKeyVaultBackend(_ core.ServerConfig) error {
134+
func (client *Crypto) setupAzureKeyVaultBackend(_ core.ServerConfig) (spi.Storage, error) {
133135
log.Logger().Debug("Setting up Azure Key Vault backend for storage of private key material.")
134-
azureBackend, err := azure.New(client.config.AzureKeyVault)
135-
if err != nil {
136-
return err
137-
}
138-
client.backend = spi.NewValidatedKIDBackendWrapper(azureBackend, spi.KidPattern)
139-
return nil
136+
return azure.New(client.config.AzureKeyVault)
140137
}
141138

142139
// List returns the KIDs of the private keys that are present in the key store.
@@ -163,24 +160,33 @@ func (client *Crypto) List(ctx context.Context) []string {
163160
func (client *Crypto) Configure(config core.ServerConfig) error {
164161
client.db = client.storage.GetSQLDatabase()
165162

163+
var backend spi.Storage
164+
var err error
166165
switch client.config.Storage {
167166
case fs.StorageType:
168-
return client.setupFSBackend(config)
167+
backend, err = client.setupFSBackend(config)
169168
case vault.StorageType:
170-
return client.setupVaultBackend(config)
169+
backend, err = client.setupVaultBackend(config)
171170
case azure.StorageType:
172-
return client.setupAzureKeyVaultBackend(config)
171+
backend, err = client.setupAzureKeyVaultBackend(config)
173172
case external.StorageType:
174-
return client.setupStorageAPIBackend()
173+
backend, err = client.setupStorageAPIBackend()
175174
case "":
176175
if config.Strictmode {
177176
return errors.New("backend must be explicitly set in strict mode")
178177
}
179178
// default to file system and run this setup again
180-
return client.setupFSBackend(config)
179+
backend, err = client.setupFSBackend(config)
181180
default:
182181
return fmt.Errorf("invalid config for crypto.storage. Available options are: vaultkv, fs, %s(experimental)", external.StorageType)
183182
}
183+
if err != nil {
184+
return fmt.Errorf("could not setup crypto backend (type=%s): %w", client.config.Storage, err)
185+
}
186+
187+
metricsWrapper := spi.NewPrometheusWrapper(spi.NewValidatedKIDBackendWrapper(backend, spi.KidPattern))
188+
client.backend = metricsWrapper
189+
return nil
184190
}
185191

186192
func (client *Crypto) Migrate() error {

crypto/crypto_test.go

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@ package crypto
2020

2121
import (
2222
"context"
23+
"crypto"
2324
"github.com/nuts-foundation/nuts-node/audit"
2425
"github.com/nuts-foundation/nuts-node/crypto/storage/fs"
2526
"github.com/nuts-foundation/nuts-node/crypto/storage/spi"
2627
"github.com/nuts-foundation/nuts-node/storage"
2728
"github.com/nuts-foundation/nuts-node/storage/orm"
29+
"github.com/nuts-foundation/nuts-node/test"
2830
"github.com/sirupsen/logrus"
2931
"github.com/stretchr/testify/require"
3032
"net/http"
@@ -191,16 +193,20 @@ func TestCrypto_Resolve(t *testing.T) {
191193
func TestCrypto_setupBackend(t *testing.T) {
192194
directory := io.TestDirectory(t)
193195
cfg := *core.NewServerConfig()
196+
cfg.Strictmode = false
194197
cfg.Datadir = directory
195198

196199
t.Run("backends should be wrapped", func(t *testing.T) {
197-
200+
client := createCrypto(t)
201+
err := client.Configure(cfg)
202+
require.NoError(t, err)
198203
t.Run("ok - fs backend is wrapped", func(t *testing.T) {
199-
client := createCrypto(t)
200-
err := client.setupFSBackend(cfg)
201-
require.NoError(t, err)
202204
storageType := reflect.TypeOf(client.backend).String()
203-
assert.Equal(t, "spi.wrapper", storageType)
205+
assert.Equal(t, "*spi.PrometheusWrapper", storageType)
206+
})
207+
t.Run("backend is wrapped in validating wrapper", func(t *testing.T) {
208+
err := client.backend.SavePrivateKey(context.Background(), "../not-allowed", nil)
209+
assert.EqualError(t, err, "invalid key ID: ../not-allowed")
204210
})
205211

206212
t.Run("ok - vault backend is wrapped", func(t *testing.T) {
@@ -211,10 +217,10 @@ func TestCrypto_setupBackend(t *testing.T) {
211217
defer s.Close()
212218
client := createCrypto(t)
213219
client.config.Vault.Address = s.URL
214-
err := client.setupVaultBackend(cfg)
220+
err := client.Configure(cfg)
215221
require.NoError(t, err)
216222
storageType := reflect.TypeOf(client.backend).String()
217-
assert.Equal(t, "spi.wrapper", storageType)
223+
assert.Equal(t, "*spi.PrometheusWrapper", storageType)
218224
})
219225
})
220226
}
@@ -266,3 +272,28 @@ func createCrypto(t *testing.T) *Crypto {
266272
}
267273
return &c
268274
}
275+
276+
func TestCrypto_StartAndShutdown(t *testing.T) {
277+
t.Run("metrics", func(t *testing.T) {
278+
storageEngine := storage.NewTestStorageEngine(t)
279+
instance := NewCryptoInstance(storageEngine)
280+
err := instance.Configure(core.TestServerConfig(func(config *core.ServerConfig) {
281+
config.Strictmode = false
282+
}))
283+
require.NoError(t, err)
284+
err = instance.Start()
285+
require.NoError(t, err)
286+
287+
// Generate some metrics
288+
_, _, err = instance.New(audit.TestContext(), func(key crypto.PublicKey) (string, error) {
289+
return "hello", nil
290+
})
291+
require.NoError(t, err)
292+
293+
stats := test.PrometheusStats(t)
294+
assert.Contains(t, stats, "crypto_storage_op_duration_seconds_count{op=\"new_private_key\"} 1")
295+
296+
err = instance.Shutdown()
297+
assert.NoError(t, err)
298+
})
299+
}

crypto/storage/spi/wrapper.go

Lines changed: 91 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,76 +23,156 @@ import (
2323
"crypto"
2424
"fmt"
2525
"github.com/nuts-foundation/nuts-node/core"
26+
"github.com/prometheus/client_golang/prometheus"
2627
"regexp"
28+
"time"
2729
)
2830

2931
// wrapper wraps a Storage backend and checks the validity of the kid on each of the relevant functions before
3032
// forwarding the call to the wrapped backend.
31-
type wrapper struct {
33+
type validationWrapper struct {
3234
kidPattern *regexp.Regexp
3335
wrappedBackend Storage
3436
}
3537

36-
func (w wrapper) Name() string {
38+
func (w validationWrapper) Name() string {
3739
return w.wrappedBackend.Name()
3840
}
3941

40-
func (w wrapper) CheckHealth() map[string]core.Health {
42+
func (w validationWrapper) CheckHealth() map[string]core.Health {
4143
return w.wrappedBackend.CheckHealth()
4244
}
4345

4446
// NewValidatedKIDBackendWrapper creates a new wrapper for storage backends.
4547
// Every call to the backend which takes a kid as param, gets the kid validated against the provided kidPattern.
4648
func NewValidatedKIDBackendWrapper(backend Storage, kidPattern *regexp.Regexp) Storage {
47-
return wrapper{
49+
return validationWrapper{
4850
kidPattern: kidPattern,
4951
wrappedBackend: backend,
5052
}
5153
}
5254

53-
func (w wrapper) validateKID(kid string) error {
55+
func (w validationWrapper) validateKID(kid string) error {
5456
if !w.kidPattern.MatchString(kid) {
5557
return fmt.Errorf("invalid key ID: %s", kid)
5658
}
5759
return nil
5860
}
5961

60-
func (w wrapper) GetPrivateKey(ctx context.Context, keyName string, version string) (crypto.Signer, error) {
62+
func (w validationWrapper) GetPrivateKey(ctx context.Context, keyName string, version string) (crypto.Signer, error) {
6163
if err := w.validateKID(keyName); err != nil {
6264
return nil, err
6365
}
6466
return w.wrappedBackend.GetPrivateKey(ctx, keyName, version)
6567
}
6668

67-
func (w wrapper) PrivateKeyExists(ctx context.Context, keyName string, version string) (bool, error) {
69+
func (w validationWrapper) PrivateKeyExists(ctx context.Context, keyName string, version string) (bool, error) {
6870
if err := w.validateKID(keyName); err != nil {
6971
return false, err
7072
}
7173
return w.wrappedBackend.PrivateKeyExists(ctx, keyName, version)
7274
}
7375

74-
func (w wrapper) SavePrivateKey(ctx context.Context, kid string, key crypto.PrivateKey) error {
76+
func (w validationWrapper) SavePrivateKey(ctx context.Context, kid string, key crypto.PrivateKey) error {
7577
if err := w.validateKID(kid); err != nil {
7678
return err
7779
}
7880
return w.wrappedBackend.SavePrivateKey(ctx, kid, key)
7981
}
8082

81-
func (w wrapper) DeletePrivateKey(ctx context.Context, keyName string) error {
83+
func (w validationWrapper) DeletePrivateKey(ctx context.Context, keyName string) error {
8284
if err := w.validateKID(keyName); err != nil {
8385
return err
8486
}
8587
return w.wrappedBackend.DeletePrivateKey(ctx, keyName)
8688
}
8789

88-
func (w wrapper) ListPrivateKeys(ctx context.Context) []KeyNameVersion {
90+
func (w validationWrapper) ListPrivateKeys(ctx context.Context) []KeyNameVersion {
8991
return w.wrappedBackend.ListPrivateKeys(ctx)
9092
}
9193

92-
func (w wrapper) NewPrivateKey(ctx context.Context, keyName string) (crypto.PublicKey, string, error) {
94+
func (w validationWrapper) NewPrivateKey(ctx context.Context, keyName string) (crypto.PublicKey, string, error) {
9395
publicKey, version, err := w.wrappedBackend.NewPrivateKey(ctx, keyName)
9496
if err != nil {
9597
return nil, "", err
9698
}
9799
return publicKey, version, err
98100
}
101+
102+
func NewPrometheusWrapper(backend Storage) *PrometheusWrapper {
103+
return &PrometheusWrapper{
104+
wrappedBackend: backend,
105+
opDurationMetric: prometheus.NewHistogramVec(prometheus.HistogramOpts{
106+
Name: "crypto_storage_op_duration_seconds",
107+
Help: "Duration of crypto storage operations in seconds (experimental, may be removed without notice)",
108+
Buckets: []float64{0.01, 0.05, 0.1, .5, 1, 2},
109+
}, []string{"op"}),
110+
}
111+
}
112+
113+
var _ Storage = (*PrometheusWrapper)(nil)
114+
115+
type PrometheusWrapper struct {
116+
wrappedBackend Storage
117+
opDurationMetric *prometheus.HistogramVec
118+
}
119+
120+
func (p PrometheusWrapper) Collectors() []prometheus.Collector {
121+
return []prometheus.Collector{p.opDurationMetric}
122+
}
123+
124+
func (p PrometheusWrapper) Name() string {
125+
return p.wrappedBackend.Name()
126+
}
127+
128+
func (p PrometheusWrapper) CheckHealth() map[string]core.Health {
129+
return p.wrappedBackend.CheckHealth()
130+
}
131+
132+
func (p PrometheusWrapper) NewPrivateKey(ctx context.Context, keyName string) (crypto.PublicKey, string, error) {
133+
start := time.Now()
134+
defer func() {
135+
p.opDurationMetric.WithLabelValues("new_private_key").Observe(time.Since(start).Seconds())
136+
}()
137+
return p.wrappedBackend.NewPrivateKey(ctx, keyName)
138+
}
139+
140+
func (p PrometheusWrapper) GetPrivateKey(ctx context.Context, keyName string, version string) (crypto.Signer, error) {
141+
start := time.Now()
142+
defer func() {
143+
p.opDurationMetric.WithLabelValues("get_private_key").Observe(time.Since(start).Seconds())
144+
}()
145+
return p.wrappedBackend.GetPrivateKey(ctx, keyName, version)
146+
}
147+
148+
func (p PrometheusWrapper) PrivateKeyExists(ctx context.Context, keyName string, version string) (bool, error) {
149+
start := time.Now()
150+
defer func() {
151+
p.opDurationMetric.WithLabelValues("private_key_exists").Observe(time.Since(start).Seconds())
152+
}()
153+
return p.wrappedBackend.PrivateKeyExists(ctx, keyName, version)
154+
}
155+
156+
func (p PrometheusWrapper) SavePrivateKey(ctx context.Context, keyname string, key crypto.PrivateKey) error {
157+
start := time.Now()
158+
defer func() {
159+
p.opDurationMetric.WithLabelValues("save_private_key").Observe(time.Since(start).Seconds())
160+
}()
161+
return p.wrappedBackend.SavePrivateKey(ctx, keyname, key)
162+
}
163+
164+
func (p PrometheusWrapper) ListPrivateKeys(ctx context.Context) []KeyNameVersion {
165+
start := time.Now()
166+
defer func() {
167+
p.opDurationMetric.WithLabelValues("list_private_keys").Observe(time.Since(start).Seconds())
168+
}()
169+
return p.wrappedBackend.ListPrivateKeys(ctx)
170+
}
171+
172+
func (p PrometheusWrapper) DeletePrivateKey(ctx context.Context, keyName string) error {
173+
start := time.Now()
174+
defer func() {
175+
p.opDurationMetric.WithLabelValues("delete_private_key").Observe(time.Since(start).Seconds())
176+
}()
177+
return p.wrappedBackend.DeletePrivateKey(ctx, keyName)
178+
}

0 commit comments

Comments
 (0)