From 36becfad3cdf1648ded5f636e80c4a63b4e50681 Mon Sep 17 00:00:00 2001 From: Rob Best Date: Fri, 30 Apr 2021 14:53:55 +0100 Subject: [PATCH 1/2] metrics: initialize counters The device names are static so we can initialize some of the exported counter vecs with 0 values. Also unexport some methods which are only used in this package. --- main.go | 20 +------------------- metrics.go | 46 ++++++++++++++++++++++++++++++++++++---------- runner.go | 6 +++--- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/main.go b/main.go index e1dfb58..1e827bd 100644 --- a/main.go +++ b/main.go @@ -9,10 +9,8 @@ import ( "regexp" "strings" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.zx2c4.com/wireguard/wgctrl" - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/utilitywarehouse/semaphore-wireguard/kube" "github.com/utilitywarehouse/semaphore-wireguard/log" @@ -107,7 +105,7 @@ func main() { } }() - makeMetricsCollector(wgMetricsClient, wgDeviceNames) + registerMetrics(wgMetricsClient, wgDeviceNames) listenAndServe(runners) // Stop runners before finishing @@ -158,22 +156,6 @@ func makeRunner(homeClient kubernetes.Interface, localName string, rConf *remote return r, wgDeviceName, nil } -func makeMetricsCollector(wgMetricsClient *wgctrl.Client, wgDeviceNames []string) { - mc := newMetricsCollector(func() ([]*wgtypes.Device, error) { - var devices []*wgtypes.Device - for _, name := range wgDeviceNames { - device, err := wgMetricsClient.Device(name) - if err != nil { - return nil, err - } - devices = append(devices, device) - } - return devices, nil - }) - prometheus.MustRegister(mc) - -} - func listenAndServe(runners []*Runner) { mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) diff --git a/metrics.go b/metrics.go index 9b9a3da..bce23f0 100644 --- a/metrics.go +++ b/metrics.go @@ -3,6 +3,7 @@ package main import ( "github.com/prometheus/client_golang/prometheus" "github.com/utilitywarehouse/semaphore-wireguard/log" + "golang.zx2c4.com/wireguard/wgctrl" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" ) @@ -30,6 +31,37 @@ var ( ) ) +// registerMetrics registers all the prometheus collectors for this package +func registerMetrics(wgMetricsClient *wgctrl.Client, wgDeviceNames []string) { + // Initialize counters with a value of 0 + for _, d := range wgDeviceNames { + for _, s := range []string{"0", "1"} { + syncPeersAttempt.With(prometheus.Labels{"device": d, "success": s}) + } + syncQueueFullFailures.With(prometheus.Labels{"device": d}) + syncRequeue.With(prometheus.Labels{"device": d}) + } + + mc := newMetricsCollector(func() ([]*wgtypes.Device, error) { + var devices []*wgtypes.Device + for _, name := range wgDeviceNames { + device, err := wgMetricsClient.Device(name) + if err != nil { + return nil, err + } + devices = append(devices, device) + } + return devices, nil + }) + + prometheus.MustRegister( + mc, + syncPeersAttempt, + syncQueueFullFailures, + syncRequeue, + ) +} + // A collector is a prometheus.Collector for a WireGuard device. type collector struct { DeviceInfo *prometheus.Desc @@ -42,13 +74,7 @@ type collector struct { devices func() ([]*wgtypes.Device, error) // to allow testing } -func init() { - prometheus.MustRegister(syncPeersAttempt) - prometheus.MustRegister(syncQueueFullFailures) - prometheus.MustRegister(syncRequeue) -} - -// NewMetricsCollector constructs a prometheus.Collector to collect metrics for +// newMetricsCollector constructs a prometheus.Collector to collect metrics for // all present wg devices and correlate with user if possible func newMetricsCollector(devices func() ([]*wgtypes.Device, error)) prometheus.Collector { // common labels for all metrics @@ -182,7 +208,7 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { } } -func MetricsSyncPeerAttempt(device string, err error) { +func metricsSyncPeerAttempt(device string, err error) { s := "1" if err != nil { s = "0" @@ -193,13 +219,13 @@ func MetricsSyncPeerAttempt(device string, err error) { }).Inc() } -func MetricsIncSyncQueueFullFailures(device string) { +func metricsIncSyncQueueFullFailures(device string) { syncQueueFullFailures.With(prometheus.Labels{ "device": device, }).Inc() } -func MetricsIncSyncRequeue(device string) { +func metricsIncSyncRequeue(device string) { syncRequeue.With(prometheus.Labels{ "device": device, }).Inc() diff --git a/runner.go b/runner.go index 07d236b..e12e7a4 100644 --- a/runner.go +++ b/runner.go @@ -123,7 +123,7 @@ func (r *Runner) syncLoop() { continue } err := r.syncPeers() - MetricsSyncPeerAttempt(r.device.Name(), err) + metricsSyncPeerAttempt(r.device.Name(), err) if err != nil { log.Logger.Warn("Failed to sync wg peers", "err", err) r.requeuePeersSync() @@ -165,7 +165,7 @@ func (r *Runner) enqueuePeersSync() { log.Logger.Debug("Sync task queued") case <-time.After(5 * time.Second): log.Logger.Error("Timed out trying to queue a sync action for netset, sync queue is full") - MetricsIncSyncQueueFullFailures(r.device.Name()) + metricsIncSyncQueueFullFailures(r.device.Name()) r.requeuePeersSync() } } @@ -174,7 +174,7 @@ func (r *Runner) requeuePeersSync() { log.Logger.Debug("Requeueing peers sync task") go func() { time.Sleep(1) - MetricsIncSyncRequeue(r.device.Name()) + metricsIncSyncRequeue(r.device.Name()) r.enqueuePeersSync() }() } From 1d0f31746985ca2d54953a901353216275a78dd4 Mon Sep 17 00:00:00 2001 From: Rob Best Date: Fri, 30 Apr 2021 14:57:51 +0100 Subject: [PATCH 2/2] metrics: promtool check metrics --- metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metrics.go b/metrics.go index bce23f0..6ee8afa 100644 --- a/metrics.go +++ b/metrics.go @@ -10,21 +10,21 @@ import ( var ( syncPeersAttempt = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "semaphore_wg_sync_peers", + Name: "semaphore_wg_sync_peers_total", Help: "Counts runners' attempts to sync peers.", }, []string{"device", "success"}, ) syncQueueFullFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "semaphore_wg_sync_queue_full_failures", + Name: "semaphore_wg_sync_queue_full_failures_total", Help: "Number of times a sync task was not added to the sync queue in time because the queue was full.", }, []string{"device"}, ) syncRequeue = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "semaphore_wg_sync_requeue", + Name: "semaphore_wg_sync_requeue_total", Help: "Number of attempts to requeue a sync.", }, []string{"device"},