Skip to content

Commit 229d8ac

Browse files
jotakjotak
authored
Update FLP, use provided enums in API (netobserv#594)
1 parent 485e361 commit 229d8ac

38 files changed

+558
-472
lines changed

‎controllers/consoleplugin/config/config.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package config
22

33
import (
4+
"github.com/netobserv/flowlogs-pipeline/pkg/api"
45
flowslatest "github.com/netobserv/network-observability-operator/apis/flowcollector/v1beta2"
56
)
67

@@ -77,12 +78,12 @@ type Deduper struct {
7778
}
7879

7980
type FrontendConfig struct {
80-
RecordTypes []string `yaml:"recordTypes" json:"recordTypes"`
81-
Columns []ColumnConfig `yaml:"columns" json:"columns"`
82-
Sampling int `yaml:"sampling" json:"sampling"`
83-
Features []string `yaml:"features" json:"features"`
84-
Deduper Deduper `yaml:"deduper" json:"deduper"`
85-
Fields []FieldConfig `yaml:"fields" json:"fields"`
81+
RecordTypes []api.ConnTrackOutputRecordTypeEnum `yaml:"recordTypes" json:"recordTypes"`
82+
Columns []ColumnConfig `yaml:"columns" json:"columns"`
83+
Sampling int `yaml:"sampling" json:"sampling"`
84+
Features []string `yaml:"features" json:"features"`
85+
Deduper Deduper `yaml:"deduper" json:"deduper"`
86+
Fields []FieldConfig `yaml:"fields" json:"fields"`
8687

8788
PortNaming flowslatest.ConsolePluginPortConfig `yaml:"portNaming,omitempty" json:"portNaming,omitempty"`
8889
Filters []FilterConfig `yaml:"filters,omitempty" json:"filters,omitempty"`

‎controllers/consoleplugin/consoleplugin_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"k8s.io/utils/ptr"
1414
"sigs.k8s.io/yaml"
1515

16+
"github.com/netobserv/flowlogs-pipeline/pkg/api"
1617
flowslatest "github.com/netobserv/network-observability-operator/apis/flowcollector/v1beta2"
1718
config "github.com/netobserv/network-observability-operator/controllers/consoleplugin/config"
1819
"github.com/netobserv/network-observability-operator/controllers/constants"
@@ -321,7 +322,7 @@ func TestConfigMapContent(t *testing.T) {
321322
assert.Equal(config.Loki.StatusURL, "https://lokistack-query-frontend-http.ls-namespace.svc:3100/")
322323

323324
// frontend params
324-
assert.Equal(config.Frontend.RecordTypes, []string{"flowLog"})
325+
assert.Equal(config.Frontend.RecordTypes, []api.ConnTrackOutputRecordTypeEnum{api.ConnTrackFlowLog})
325326
assert.Empty(config.Frontend.Features)
326327
assert.NotEmpty(config.Frontend.Columns)
327328
assert.NotEmpty(config.Frontend.Filters)

‎controllers/constants/constants.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,6 @@ const (
3333

3434
TokensPath = "/var/run/secrets/tokens/"
3535

36-
FlowLogType = "flowLog"
37-
NewConnectionType = "newConnection"
38-
HeartbeatType = "heartbeat"
39-
EndConnectionType = "endConnection"
40-
4136
ClusterNameLabelName = "K8S_ClusterName"
4237

4338
MonitoringNamespace = "openshift-monitoring"

‎controllers/flp/flp_common_objects.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ func (b *builder) NewKafkaPipeline() PipelineBuilder {
126126
return b.initPipeline(config.NewKafkaPipeline("kafka-read", api.IngestKafka{
127127
Brokers: []string{b.desired.Kafka.Address},
128128
Topic: b.desired.Kafka.Topic,
129-
GroupId: b.name(), // Without groupid, each message is delivered to each consumers
129+
GroupID: b.name(), // Without groupid, each message is delivered to each consumers
130130
Decoder: decoder,
131131
TLS: getKafkaTLS(&b.desired.Kafka.TLS, "kafka-cert", &b.volumes),
132132
SASL: getKafkaSASL(&b.desired.Kafka.SASL, "kafka-ingest", &b.volumes),

‎controllers/flp/flp_pipeline_builder.go

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -59,23 +59,23 @@ func (b *PipelineBuilder) AddProcessorStages() error {
5959
// enrich stage (transform) configuration
6060
enrichedStage := lastStage.TransformNetwork("enrich", api.TransformNetwork{
6161
Rules: api.NetworkTransformRules{{
62-
Type: api.AddKubernetesRuleType,
62+
Type: api.NetworkAddKubernetes,
6363
Kubernetes: &api.K8sRule{
6464
Input: "SrcAddr",
6565
Output: "SrcK8S",
6666
AddZone: addZone,
6767
},
6868
}, {
69-
Type: api.AddKubernetesRuleType,
69+
Type: api.NetworkAddKubernetes,
7070
Kubernetes: &api.K8sRule{
7171
Input: "DstAddr",
7272
Output: "DstK8S",
7373
AddZone: addZone,
7474
},
7575
}, {
76-
Type: api.ReinterpretDirectionRuleType,
76+
Type: api.NetworkReinterpretDirection,
7777
}, {
78-
Type: api.OpAddKubernetesInfra,
78+
Type: api.NetworkAddKubernetesInfra,
7979
KubernetesInfra: &api.K8sInfraRule{
8080
Inputs: []string{
8181
"SrcAddr",
@@ -193,21 +193,21 @@ func (b *PipelineBuilder) AddProcessorStages() error {
193193
func flowMetricToFLP(flowMetric *metricslatest.FlowMetricSpec) (*api.MetricsItem, error) {
194194
m := &api.MetricsItem{
195195
Name: flowMetric.MetricName,
196-
Type: strings.ToLower(string(flowMetric.Type)),
196+
Type: api.MetricEncodeOperationEnum(strings.ToLower(string(flowMetric.Type))),
197197
Filters: []api.MetricsFilter{},
198198
Labels: flowMetric.Labels,
199199
ValueKey: flowMetric.ValueField,
200200
}
201201
for _, f := range flowMetric.Filters {
202-
m.Filters = append(m.Filters, api.MetricsFilter{Key: f.Field, Value: f.Value, Type: conversion.PascalToLower(string(f.MatchType), '_')})
202+
m.Filters = append(m.Filters, api.MetricsFilter{Key: f.Field, Value: f.Value, Type: api.MetricFilterEnum(conversion.PascalToLower(string(f.MatchType), '_'))})
203203
}
204204
if !flowMetric.IncludeDuplicates {
205-
m.Filters = append(m.Filters, api.MetricsFilter{Key: "Duplicate", Value: "true", Type: api.PromFilterNotEqual})
205+
m.Filters = append(m.Filters, api.MetricsFilter{Key: "Duplicate", Value: "true", Type: api.MetricFilterNotEqual})
206206
}
207207
if flowMetric.Direction == metricslatest.Egress {
208-
m.Filters = append(m.Filters, api.MetricsFilter{Key: "FlowDirection", Value: "1|2", Type: api.PromFilterRegex})
208+
m.Filters = append(m.Filters, api.MetricsFilter{Key: "FlowDirection", Value: "1|2", Type: api.MetricFilterRegex})
209209
} else if flowMetric.Direction == metricslatest.Ingress {
210-
m.Filters = append(m.Filters, api.MetricsFilter{Key: "FlowDirection", Value: "0|2", Type: api.PromFilterRegex})
210+
m.Filters = append(m.Filters, api.MetricsFilter{Key: "FlowDirection", Value: "0|2", Type: api.MetricFilterRegex})
211211
}
212212
for _, b := range flowMetric.Buckets {
213213
f, err := strconv.ParseFloat(b, 64)
@@ -377,9 +377,11 @@ func (b *PipelineBuilder) addTransformFilter(lastStage config.PipelineBuilderSta
377377
if clusterName != "" {
378378
transformFilterRules = []api.TransformFilterRule{
379379
{
380-
Input: constants.ClusterNameLabelName,
381-
Type: "add_field_if_doesnt_exist",
382-
Value: clusterName,
380+
Type: api.AddFieldIfDoesntExist,
381+
AddFieldIfDoesntExist: &api.TransformFilterGenericRule{
382+
Input: constants.ClusterNameLabelName,
383+
Value: clusterName,
384+
},
383385
},
384386
}
385387
}
@@ -452,9 +454,9 @@ func getKafkaSASL(sasl *flowslatest.SASLConfig, volumePrefix string, volumes *vo
452454
if !helper.UseSASL(sasl) {
453455
return nil
454456
}
455-
t := "plain"
457+
t := api.SASLPlain
456458
if sasl.Type == flowslatest.SASLScramSHA512 {
457-
t = "scramSHA512"
459+
t = api.SASLScramSHA512
458460
}
459461
idPath := volumes.AddVolume(&sasl.ClientIDReference, volumePrefix+"-sasl-id")
460462
secretPath := volumes.AddVolume(&sasl.ClientSecretReference, volumePrefix+"-sasl-secret")

‎controllers/flp/metrics_api_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,8 @@ func TestFlowMetricToFLP(t *testing.T) {
9191
Name: "m_1",
9292
Type: "counter",
9393
Filters: []api.MetricsFilter{
94-
{Key: "f", Value: "v", Type: api.PromFilterEqual},
95-
{Key: "Duplicate", Value: "true", Type: api.PromFilterNotEqual},
94+
{Key: "f", Value: "v", Type: api.MetricFilterEqual},
95+
{Key: "Duplicate", Value: "true", Type: api.MetricFilterNotEqual},
9696
},
9797
ValueKey: "val",
9898
Labels: []string{"by_field"},
@@ -102,9 +102,9 @@ func TestFlowMetricToFLP(t *testing.T) {
102102
Name: "m_2",
103103
Type: "histogram",
104104
Filters: []api.MetricsFilter{
105-
{Key: "f", Value: "v", Type: api.PromFilterRegex},
106-
{Key: "f2", Type: api.PromFilterAbsence},
107-
{Key: "FlowDirection", Value: "1|2", Type: api.PromFilterRegex},
105+
{Key: "f", Value: "v", Type: api.MetricFilterRegex},
106+
{Key: "f2", Type: api.MetricFilterAbsence},
107+
{Key: "FlowDirection", Value: "1|2", Type: api.MetricFilterRegex},
108108
},
109109
Labels: []string{"by_field"},
110110
Buckets: []float64{1, 5, 10, 50, 100},

‎go.mod

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ toolchain go1.21.7
66

77
require (
88
github.com/go-logr/logr v1.4.1
9-
github.com/netobserv/flowlogs-pipeline v0.1.12-0.20240305083238-24bf8cec8807
9+
github.com/netobserv/flowlogs-pipeline v0.1.12-0.20240322124726-d2b2352bfe0f
1010
github.com/onsi/ginkgo/v2 v2.16.0
1111
github.com/onsi/gomega v1.31.1
1212
github.com/openshift/api v0.0.0-20220112145620-704957ce4980
@@ -65,10 +65,10 @@ require (
6565
github.com/stretchr/objx v0.5.2 // indirect
6666
go.uber.org/multierr v1.11.0 // indirect
6767
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
68-
golang.org/x/net v0.21.0 // indirect
68+
golang.org/x/net v0.22.0 // indirect
6969
golang.org/x/oauth2 v0.16.0 // indirect
70-
golang.org/x/sys v0.17.0 // indirect
71-
golang.org/x/term v0.17.0 // indirect
70+
golang.org/x/sys v0.18.0 // indirect
71+
golang.org/x/term v0.18.0 // indirect
7272
golang.org/x/text v0.14.0 // indirect
7373
golang.org/x/time v0.5.0 // indirect
7474
golang.org/x/tools v0.17.0 // indirect

0 commit comments

Comments
 (0)