Skip to content

Commit ede3a92

Browse files
scrayosscrayos
committed
add monitoring for seaweed components
1 parent ae0f057 commit ede3a92

17 files changed

+431
-1054
lines changed

‎api/v1/seaweed_types.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ type MasterSpec struct {
122122
// Config in raw toml string
123123
Config *string `json:"config,omitempty"`
124124

125+
// MetricsPort is the port that the prometheus metrics export listens on
126+
MetricsPort *int32 `json:"metricsPort,omitempty"`
127+
125128
// Master-specific settings
126129

127130
VolumePreallocate *bool `json:"volumePreallocate,omitempty"`
@@ -145,6 +148,9 @@ type VolumeSpec struct {
145148

146149
StorageClassName *string `json:"storageClassName,omitempty"`
147150

151+
// MetricsPort is the port that the prometheus metrics export listens on
152+
MetricsPort *int32 `json:"metricsPort,omitempty"`
153+
148154
// Volume-specific settings
149155

150156
CompactionMBps *int32 `json:"compactionMBps,omitempty"`
@@ -168,6 +174,9 @@ type FilerSpec struct {
168174
// Config in raw toml string
169175
Config *string `json:"config,omitempty"`
170176

177+
// MetricsPort is the port that the prometheus metrics export listens on
178+
MetricsPort *int32 `json:"metricsPort,omitempty"`
179+
171180
// Filer-specific settings
172181

173182
MaxMB *int32 `json:"maxMB,omitempty"`

‎config/crd/bases/seaweed.seaweedfs.com_seaweeds.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1413,6 +1413,9 @@ spec:
14131413
maxMB:
14141414
format: int32
14151415
type: integer
1416+
metricsPort:
1417+
format: int32
1418+
type: integer
14161419
nodeSelector:
14171420
additionalProperties:
14181421
type: string
@@ -2337,6 +2340,9 @@ spec:
23372340
description: 'Limits describes the maximum amount of compute resources
23382341
allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
23392342
type: object
2343+
metricsPort:
2344+
format: int32
2345+
type: integer
23402346
nodeSelector:
23412347
additionalProperties:
23422348
type: string
@@ -3309,6 +3315,9 @@ spec:
33093315
maxVolumeCounts:
33103316
format: int32
33113317
type: integer
3318+
metricsPort:
3319+
format: int32
3320+
type: integer
33123321
minFreeSpacePercent:
33133322
format: int32
33143323
type: integer

‎controllers/controller_filer.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ func (r *SeaweedReconciler) ensureFilerServers(seaweedCR *seaweedv1.Seaweed) (do
3333
return
3434
}
3535

36+
if seaweedCR.Spec.Filer.MetricsPort != nil {
37+
if done, result, err = r.ensureFilerServiceMonitor(seaweedCR); done {
38+
return
39+
}
40+
}
41+
3642
return
3743
}
3844

@@ -98,6 +104,19 @@ func (r *SeaweedReconciler) ensureFilerConfigMap(seaweedCR *seaweedv1.Seaweed) (
98104
return ReconcileResult(err)
99105
}
100106

107+
func (r *SeaweedReconciler) ensureFilerServiceMonitor(seaweedCR *seaweedv1.Seaweed) (bool, ctrl.Result, error) {
108+
log := r.Log.WithValues("sw-filer-servicemonitor", seaweedCR.Name)
109+
110+
filerServiceMonitor := r.createFilerServiceMonitor(seaweedCR)
111+
if err := controllerutil.SetControllerReference(seaweedCR, filerServiceMonitor, r.Scheme); err != nil {
112+
return ReconcileResult(err)
113+
}
114+
_, err := r.CreateOrUpdateServiceMonitor(filerServiceMonitor)
115+
116+
log.Info("Get filer service monitor " + filerServiceMonitor.Name)
117+
return ReconcileResult(err)
118+
}
119+
101120
func labelsForFiler(name string) map[string]string {
102121
return map[string]string{
103122
label.ManagedByLabelKey: "seaweedfs-operator",

‎controllers/controller_filer_service.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ func (r *SeaweedReconciler) createFilerPeerService(m *seaweedv1.Seaweed) *corev1
3232
TargetPort: intstr.FromInt(seaweedv1.FilerS3Port),
3333
})
3434
}
35+
if m.Spec.Filer.MetricsPort != nil {
36+
ports = append(ports, corev1.ServicePort{
37+
Name: "filer-metrics",
38+
Protocol: corev1.Protocol("TCP"),
39+
Port: *m.Spec.Filer.MetricsPort,
40+
TargetPort: intstr.FromInt(int(*m.Spec.Filer.MetricsPort)),
41+
})
42+
}
3543

3644
dep := &corev1.Service{
3745
ObjectMeta: metav1.ObjectMeta{
@@ -76,6 +84,14 @@ func (r *SeaweedReconciler) createFilerService(m *seaweedv1.Seaweed) *corev1.Ser
7684
TargetPort: intstr.FromInt(seaweedv1.FilerS3Port),
7785
})
7886
}
87+
if m.Spec.Filer.MetricsPort != nil {
88+
ports = append(ports, corev1.ServicePort{
89+
Name: "filer-metrics",
90+
Protocol: corev1.Protocol("TCP"),
91+
Port: *m.Spec.Filer.MetricsPort,
92+
TargetPort: intstr.FromInt(int(*m.Spec.Filer.MetricsPort)),
93+
})
94+
}
7995

8096
dep := &corev1.Service{
8197
ObjectMeta: metav1.ObjectMeta{

‎controllers/controller_filer_servicemonitor.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package controllers
2+
3+
import (
4+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
5+
6+
monitorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
7+
seaweedv1 "github.com/seaweedfs/seaweedfs-operator/api/v1"
8+
)
9+
10+
func (r *SeaweedReconciler) createFilerServiceMonitor(m *seaweedv1.Seaweed) *monitorv1.ServiceMonitor {
11+
labels := labelsForFiler(m.Name)
12+
13+
dep := &monitorv1.ServiceMonitor{
14+
ObjectMeta: metav1.ObjectMeta{
15+
Name: m.Name + "-filer-metrics-monitor",
16+
Namespace: m.Namespace,
17+
Labels: labels,
18+
},
19+
Spec: monitorv1.ServiceMonitorSpec{
20+
Endpoints: []monitorv1.Endpoint{
21+
{
22+
Path: "/metrics",
23+
Port: "filer-metrics",
24+
},
25+
},
26+
Selector: metav1.LabelSelector{
27+
MatchLabels: labels,
28+
},
29+
},
30+
}
31+
32+
return dep
33+
}

‎controllers/controller_filer_statefulset.go‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,37 @@ func buildFilerStartupScript(m *seaweedv1.Seaweed) string {
2020
if m.Spec.Filer.S3 {
2121
commands = append(commands, "-s3")
2222
}
23+
if m.Spec.Filer.MetricsPort != nil {
24+
commands = append(commands, fmt.Sprintf("-metricsPort=%d", *m.Spec.Filer.MetricsPort))
25+
}
2326

2427
return strings.Join(commands, " ")
2528
}
2629

2730
func (r *SeaweedReconciler) createFilerStatefulSet(m *seaweedv1.Seaweed) *appsv1.StatefulSet {
2831
labels := labelsForFiler(m.Name)
32+
ports := []corev1.ContainerPort{
33+
{
34+
ContainerPort: seaweedv1.FilerHTTPPort,
35+
Name: "filer-http",
36+
},
37+
{
38+
ContainerPort: seaweedv1.FilerGRPCPort,
39+
Name: "filer-grpc",
40+
},
41+
}
42+
if m.Spec.Filer.S3 {
43+
ports = append(ports, corev1.ContainerPort{
44+
ContainerPort: seaweedv1.FilerS3Port,
45+
Name: "filer-s3",
46+
})
47+
}
48+
if m.Spec.Filer.MetricsPort != nil {
49+
ports = append(ports, corev1.ContainerPort{
50+
ContainerPort: *m.Spec.Filer.MetricsPort,
51+
Name: "filer-metrics",
52+
})
53+
}
2954
replicas := int32(m.Spec.Filer.Replicas)
3055
rollingUpdatePartition := int32(0)
3156
enableServiceLinks := false
@@ -61,20 +86,7 @@ func (r *SeaweedReconciler) createFilerStatefulSet(m *seaweedv1.Seaweed) *appsv1
6186
"-ec",
6287
buildFilerStartupScript(m),
6388
},
64-
Ports: []corev1.ContainerPort{
65-
{
66-
ContainerPort: seaweedv1.FilerHTTPPort,
67-
Name: "filer-http",
68-
},
69-
{
70-
ContainerPort: seaweedv1.FilerGRPCPort,
71-
Name: "filer-grpc",
72-
},
73-
{
74-
ContainerPort: seaweedv1.FilerS3Port,
75-
Name: "filer-s3",
76-
},
77-
},
89+
Ports: ports,
7890
ReadinessProbe: &corev1.Probe{
7991
ProbeHandler: corev1.ProbeHandler{
8092
HTTPGet: &corev1.HTTPGetAction{

‎controllers/controller_master.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@ func (r *SeaweedReconciler) ensureMaster(seaweedCR *seaweedv1.Seaweed) (done boo
4141
}
4242
}
4343

44+
if seaweedCR.Spec.Master.MetricsPort != nil {
45+
if done, result, err = r.ensureMasterServiceMonitor(seaweedCR); done {
46+
return
47+
}
48+
}
49+
4450
return
4551
}
4652

@@ -138,7 +144,19 @@ func (r *SeaweedReconciler) ensureMasterPeerService(seaweedCR *seaweedv1.Seaweed
138144

139145
log.Info("Get master peer service " + masterPeerService.Name)
140146
return ReconcileResult(err)
147+
}
141148

149+
func (r *SeaweedReconciler) ensureMasterServiceMonitor(seaweedCR *seaweedv1.Seaweed) (bool, ctrl.Result, error) {
150+
log := r.Log.WithValues("sw-master-servicemonitor", seaweedCR.Name)
151+
152+
masterServiceMonitor := r.createMasterServiceMonitor(seaweedCR)
153+
if err := controllerutil.SetControllerReference(seaweedCR, masterServiceMonitor, r.Scheme); err != nil {
154+
return ReconcileResult(err)
155+
}
156+
_, err := r.CreateOrUpdateServiceMonitor(masterServiceMonitor)
157+
158+
log.Info("Get master service monitor " + masterServiceMonitor.Name)
159+
return ReconcileResult(err)
142160
}
143161

144162
func labelsForMaster(name string) map[string]string {

‎controllers/controller_master_service.go‎

Lines changed: 48 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,28 @@ import (
1010

1111
func (r *SeaweedReconciler) createMasterPeerService(m *seaweedv1.Seaweed) *corev1.Service {
1212
labels := labelsForMaster(m.Name)
13+
ports := []corev1.ServicePort{
14+
{
15+
Name: "master-http",
16+
Protocol: corev1.Protocol("TCP"),
17+
Port: seaweedv1.MasterHTTPPort,
18+
TargetPort: intstr.FromInt(seaweedv1.MasterHTTPPort),
19+
},
20+
{
21+
Name: "master-grpc",
22+
Protocol: corev1.Protocol("TCP"),
23+
Port: seaweedv1.MasterGRPCPort,
24+
TargetPort: intstr.FromInt(seaweedv1.MasterGRPCPort),
25+
},
26+
}
27+
if m.Spec.Master.MetricsPort != nil {
28+
ports = append(ports, corev1.ServicePort{
29+
Name: "master-metrics",
30+
Protocol: corev1.Protocol("TCP"),
31+
Port: *m.Spec.Master.MetricsPort,
32+
TargetPort: intstr.FromInt(int(*m.Spec.Master.MetricsPort)),
33+
})
34+
}
1335

1436
dep := &corev1.Service{
1537
ObjectMeta: metav1.ObjectMeta{
@@ -23,21 +45,8 @@ func (r *SeaweedReconciler) createMasterPeerService(m *seaweedv1.Seaweed) *corev
2345
Spec: corev1.ServiceSpec{
2446
ClusterIP: "None",
2547
PublishNotReadyAddresses: true,
26-
Ports: []corev1.ServicePort{
27-
{
28-
Name: "master-http",
29-
Protocol: corev1.Protocol("TCP"),
30-
Port: seaweedv1.MasterHTTPPort,
31-
TargetPort: intstr.FromInt(seaweedv1.MasterHTTPPort),
32-
},
33-
{
34-
Name: "master-grpc",
35-
Protocol: corev1.Protocol("TCP"),
36-
Port: seaweedv1.MasterGRPCPort,
37-
TargetPort: intstr.FromInt(seaweedv1.MasterGRPCPort),
38-
},
39-
},
40-
Selector: labels,
48+
Ports: ports,
49+
Selector: labels,
4150
},
4251
}
4352
// Set master instance as the owner and controller
@@ -47,6 +56,28 @@ func (r *SeaweedReconciler) createMasterPeerService(m *seaweedv1.Seaweed) *corev
4756

4857
func (r *SeaweedReconciler) createMasterService(m *seaweedv1.Seaweed) *corev1.Service {
4958
labels := labelsForMaster(m.Name)
59+
ports := []corev1.ServicePort{
60+
{
61+
Name: "master-http",
62+
Protocol: corev1.Protocol("TCP"),
63+
Port: seaweedv1.MasterHTTPPort,
64+
TargetPort: intstr.FromInt(seaweedv1.MasterHTTPPort),
65+
},
66+
{
67+
Name: "master-grpc",
68+
Protocol: corev1.Protocol("TCP"),
69+
Port: seaweedv1.MasterGRPCPort,
70+
TargetPort: intstr.FromInt(seaweedv1.MasterGRPCPort),
71+
},
72+
}
73+
if m.Spec.Master.MetricsPort != nil {
74+
ports = append(ports, corev1.ServicePort{
75+
Name: "master-metrics",
76+
Protocol: corev1.Protocol("TCP"),
77+
Port: *m.Spec.Master.MetricsPort,
78+
TargetPort: intstr.FromInt(int(*m.Spec.Master.MetricsPort)),
79+
})
80+
}
5081

5182
dep := &corev1.Service{
5283
ObjectMeta: metav1.ObjectMeta{
@@ -59,21 +90,8 @@ func (r *SeaweedReconciler) createMasterService(m *seaweedv1.Seaweed) *corev1.Se
5990
},
6091
Spec: corev1.ServiceSpec{
6192
PublishNotReadyAddresses: true,
62-
Ports: []corev1.ServicePort{
63-
{
64-
Name: "master-http",
65-
Protocol: corev1.Protocol("TCP"),
66-
Port: seaweedv1.MasterHTTPPort,
67-
TargetPort: intstr.FromInt(seaweedv1.MasterHTTPPort),
68-
},
69-
{
70-
Name: "master-grpc",
71-
Protocol: corev1.Protocol("TCP"),
72-
Port: seaweedv1.MasterGRPCPort,
73-
TargetPort: intstr.FromInt(seaweedv1.MasterGRPCPort),
74-
},
75-
},
76-
Selector: labels,
93+
Ports: ports,
94+
Selector: labels,
7795
},
7896
}
7997

‎controllers/controller_master_servicemonitor.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package controllers
2+
3+
import (
4+
monitorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
5+
seaweedv1 "github.com/seaweedfs/seaweedfs-operator/api/v1"
6+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
7+
)
8+
9+
func (r *SeaweedReconciler) createMasterServiceMonitor(m *seaweedv1.Seaweed) *monitorv1.ServiceMonitor {
10+
labels := labelsForMaster(m.Name)
11+
12+
dep := &monitorv1.ServiceMonitor{
13+
ObjectMeta: metav1.ObjectMeta{
14+
Name: m.Name + "-master-metrics-monitor",
15+
Namespace: m.Namespace,
16+
Labels: labels,
17+
},
18+
Spec: monitorv1.ServiceMonitorSpec{
19+
Endpoints: []monitorv1.Endpoint{
20+
{
21+
Path: "/metrics",
22+
Port: "master-metrics",
23+
},
24+
},
25+
Selector: metav1.LabelSelector{
26+
MatchLabels: labels,
27+
},
28+
},
29+
}
30+
31+
return dep
32+
}

0 commit comments

Comments
 (0)