Skip to content

Commit 3033612

Browse files
committed
configutils: Enable egress configuration
Introduce the possibility of configuring egress for any `*rest.Config` obtained via `configutils.GetConfig`.
1 parent ec6ab8e commit 3033612

File tree

4 files changed

+263
-5
lines changed

4 files changed

+263
-5
lines changed

configutils/configutils.go

+106-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"os/user"
2222
"path/filepath"
2323

24+
"k8s.io/apiserver/pkg/server/egressselector"
2425
"k8s.io/client-go/rest"
2526
"k8s.io/client-go/tools/clientcmd"
2627
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -32,13 +33,44 @@ var (
3233
log = ctrl.Log.WithName("configutils")
3334
)
3435

36+
// EgressSelectionName is the name of the egress configuration to use.
37+
type EgressSelectionName string
38+
39+
const (
40+
// EgressSelectionNameControlPlane instructs to use the controlplane egress selection.
41+
EgressSelectionNameControlPlane EgressSelectionName = "controlplane"
42+
// EgressSelectionNameEtcd instructs to use the etcd egress selection.
43+
EgressSelectionNameEtcd EgressSelectionName = "etcd"
44+
// EgressSelectionNameCluster instructs to use the cluster egress selection.
45+
EgressSelectionNameCluster EgressSelectionName = "cluster"
46+
)
47+
48+
// NetworkContext returns the corresponding network context of the egress selection.
49+
func (n EgressSelectionName) NetworkContext() (egressselector.NetworkContext, error) {
50+
switch n {
51+
case EgressSelectionNameControlPlane:
52+
return egressselector.ControlPlane.AsNetworkContext(), nil
53+
case EgressSelectionNameEtcd:
54+
return egressselector.Etcd.AsNetworkContext(), nil
55+
case EgressSelectionNameCluster:
56+
return egressselector.Cluster.AsNetworkContext(), nil
57+
default:
58+
return egressselector.NetworkContext{}, fmt.Errorf("unknown egress selection name %q", n)
59+
}
60+
}
61+
3562
// GetConfigOptions are options to supply for a GetConfig call.
3663
type GetConfigOptions struct {
3764
// Context is the kubeconfig context to load.
3865
Context string
3966
// Kubeconfig is the path to a kubeconfig to load.
4067
// If unset, the '--kubeconfig' flag is used.
4168
Kubeconfig *string
69+
// EgressSelectorConfig is the path to an egress selector config to load.
70+
EgressSelectorConfig string
71+
// EgressSelectionName is the name of the egress configuration to use.
72+
// Defaults to EgressSelectionNameControlPlane.
73+
EgressSelectionName EgressSelectionName
4274
}
4375

4476
// ApplyToGetConfig implements GetConfigOption.
@@ -49,6 +81,12 @@ func (o *GetConfigOptions) ApplyToGetConfig(o2 *GetConfigOptions) {
4981
if o.Kubeconfig != nil {
5082
o2.Kubeconfig = pointer.String(*o.Kubeconfig)
5183
}
84+
if o.EgressSelectorConfig != "" {
85+
o2.EgressSelectorConfig = o.EgressSelectorConfig
86+
}
87+
if o.EgressSelectionName != "" {
88+
o2.EgressSelectionName = o.EgressSelectionName
89+
}
5290
}
5391

5492
// ApplyOptions applies all GetConfigOption tro this GetConfigOptions.
@@ -74,6 +112,20 @@ func (c Context) ApplyToGetConfig(o *GetConfigOptions) {
74112
o.Context = string(c)
75113
}
76114

115+
// EgressSelectorConfig allows specifying the path to an egress selector config to use.
116+
type EgressSelectorConfig string
117+
118+
func (c EgressSelectorConfig) ApplyToGetConfig(o *GetConfigOptions) {
119+
o.EgressSelectorConfig = string(c)
120+
}
121+
122+
type WithEgressSelectionName EgressSelectionName
123+
124+
// ApplyToGetConfig implements GetConfigOption.
125+
func (w WithEgressSelectionName) ApplyToGetConfig(o *GetConfigOptions) {
126+
o.EgressSelectionName = EgressSelectionName(w)
127+
}
128+
77129
// GetConfigOption are options to a GetConfig call.
78130
type GetConfigOption interface {
79131
// ApplyToGetConfig modifies the underlying GetConfigOptions.
@@ -105,6 +157,12 @@ func getKubeconfigFlag() string {
105157
return f.Value.String()
106158
}
107159

160+
func setGetConfigOptionsDefaults(o *GetConfigOptions) {
161+
if o.EgressSelectionName == "" {
162+
o.EgressSelectionName = EgressSelectionNameControlPlane
163+
}
164+
}
165+
108166
// GetConfig creates a *rest.Config for talking to a Kubernetes API server.
109167
// Kubeconfig / the '--kubeconfig' flag instruct to use the kubeconfig file at that location.
110168
// Otherwise, will assume running in cluster and use the cluster provided kubeconfig.
@@ -124,6 +182,7 @@ func getKubeconfigFlag() string {
124182
func GetConfig(opts ...GetConfigOption) (*rest.Config, error) {
125183
o := &GetConfigOptions{}
126184
o.ApplyOptions(opts)
185+
setGetConfigOptionsDefaults(o)
127186

128187
var kubeconfig string
129188
if o.Kubeconfig != nil {
@@ -132,9 +191,22 @@ func GetConfig(opts ...GetConfigOption) (*rest.Config, error) {
132191
kubeconfig = getKubeconfigFlag()
133192
}
134193

194+
cfg, err := loadConfig(kubeconfig, o.Context)
195+
if err != nil {
196+
return nil, fmt.Errorf("error loading config: %w", err)
197+
}
198+
199+
if err := applyEgressSelector(o.EgressSelectorConfig, o.EgressSelectionName, cfg); err != nil {
200+
return nil, fmt.Errorf("error applying egress selector: %w", err)
201+
}
202+
203+
return cfg, nil
204+
}
205+
206+
func loadConfig(kubeconfig, context string) (*rest.Config, error) {
135207
// If a flag is specified with the config location, use that
136208
if len(kubeconfig) > 0 {
137-
return loadConfigWithContext("", &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, o.Context)
209+
return loadConfigWithContext("", &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, context)
138210
}
139211

140212
// If the recommended kubeconfig env variable is not specified,
@@ -155,7 +227,39 @@ func GetConfig(opts ...GetConfigOption) (*rest.Config, error) {
155227
loadingRules.Precedence = append(loadingRules.Precedence, filepath.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName))
156228
}
157229

158-
return loadConfigWithContext("", loadingRules, o.Context)
230+
return loadConfigWithContext("", loadingRules, context)
231+
}
232+
233+
func applyEgressSelector(egressSelectorConfig string, egressSelectionName EgressSelectionName, cfg *rest.Config) error {
234+
if egressSelectorConfig == "" {
235+
return nil
236+
}
237+
238+
networkContext, err := egressSelectionName.NetworkContext()
239+
if err != nil {
240+
return fmt.Errorf("error obtaining network context: %w", err)
241+
}
242+
243+
egressSelectorCfg, err := egressselector.ReadEgressSelectorConfiguration(egressSelectorConfig)
244+
if err != nil {
245+
return fmt.Errorf("error reading egress selector configuration: %w", err)
246+
}
247+
248+
egressSelector, err := egressselector.NewEgressSelector(egressSelectorCfg)
249+
if err != nil {
250+
return fmt.Errorf("error creating egress selector: %w", err)
251+
}
252+
253+
dial, err := egressSelector.Lookup(networkContext)
254+
if err != nil {
255+
return fmt.Errorf("error looking up network context %s: %w", networkContext.EgressSelectionName.String(), err)
256+
}
257+
if dial == nil {
258+
return fmt.Errorf("no dialer for network context %s", networkContext.EgressSelectionName.String())
259+
}
260+
261+
cfg.Dial = dial
262+
return nil
159263
}
160264

161265
// GetConfigOrDie creates a *rest.Config for talking to a Kubernetes apiserver.

configutils/configutils_test.go

+67-3
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,16 @@ package configutils
1717
import (
1818
"flag"
1919
"os"
20+
"path/filepath"
2021

2122
"github.com/onsi/ginkgo/v2"
2223
. "github.com/onsi/gomega"
2324
corev1 "k8s.io/api/core/v1"
25+
"k8s.io/apiextensions-apiserver/pkg/apiserver"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/runtime"
28+
"k8s.io/apimachinery/pkg/runtime/serializer/json"
29+
apiserverv1beta1 "k8s.io/apiserver/pkg/apis/apiserver/v1beta1"
2430
"k8s.io/client-go/rest"
2531
"k8s.io/client-go/tools/clientcmd"
2632
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -31,11 +37,25 @@ func setKubeconfigFlag(kubeconfig string) {
3137
}
3238

3339
var _ = ginkgo.Describe("Configutils", func() {
40+
apiServerSerializer := json.NewSerializerWithOptions(
41+
json.DefaultMetaFactory,
42+
apiserver.Scheme,
43+
apiserver.Scheme,
44+
json.SerializerOptions{
45+
Yaml: true,
46+
},
47+
)
48+
3449
ginkgo.Describe("GetConfig", func() {
50+
3551
var (
36-
apiConfig *clientcmdapi.Config
37-
config *rest.Config
38-
otherConfig *rest.Config
52+
apiConfig *clientcmdapi.Config
53+
egressConfig *apiserverv1beta1.EgressSelectorConfiguration
54+
config *rest.Config
55+
otherConfig *rest.Config
56+
57+
configFile string
58+
egressConfigFile string
3959
)
4060
ginkgo.BeforeEach(func() {
4161
apiConfig = &clientcmdapi.Config{
@@ -71,6 +91,31 @@ var _ = ginkgo.Describe("Configutils", func() {
7191
otherConfig = &rest.Config{
7292
Host: "http://other.example.org",
7393
}
94+
egressConfig = &apiserverv1beta1.EgressSelectorConfiguration{
95+
TypeMeta: metav1.TypeMeta{
96+
APIVersion: apiserverv1beta1.SchemeGroupVersion.String(),
97+
Kind: "EgressSelectorConfiguration",
98+
},
99+
EgressSelections: []apiserverv1beta1.EgressSelection{
100+
{
101+
Name: "controlplane",
102+
Connection: apiserverv1beta1.Connection{
103+
ProxyProtocol: apiserverv1beta1.ProtocolDirect,
104+
},
105+
},
106+
},
107+
}
108+
109+
tempDir := ginkgo.GinkgoT().TempDir()
110+
111+
configFile = filepath.Join(tempDir, "kubeconfig")
112+
Expect(clientcmd.WriteToFile(*apiConfig, configFile)).To(Succeed())
113+
114+
egressConfigData, err := runtime.Encode(apiServerSerializer, egressConfig)
115+
Expect(err).NotTo(HaveOccurred())
116+
117+
egressConfigFile = filepath.Join(tempDir, "egress-config.yaml")
118+
Expect(os.WriteFile(egressConfigFile, egressConfigData, 0666)).To(Succeed())
74119
})
75120

76121
ginkgo.It("should load the config at the kubeconfig path", func() {
@@ -122,5 +167,24 @@ var _ = ginkgo.Describe("Configutils", func() {
122167
_, err := GetConfig()
123168
Expect(err).To(HaveOccurred())
124169
})
170+
171+
ginkgo.It("should load the kubeconfig and apply the egress selector", func() {
172+
cfg, err := GetConfig(Kubeconfig(configFile), EgressSelectorConfig(egressConfigFile))
173+
Expect(err).NotTo(HaveOccurred())
174+
Expect(cfg.Dial).NotTo(BeNil())
175+
})
176+
177+
ginkgo.It("should error if the egress selector config file does not exist", func() {
178+
_, err := GetConfig(Kubeconfig(configFile), EgressSelectorConfig("should-never-exist"))
179+
Expect(err).To(HaveOccurred())
180+
})
181+
182+
ginkgo.It("should error if the egress context does not exist", func() {
183+
_, err := GetConfig(Kubeconfig(configFile),
184+
EgressSelectorConfig(egressConfigFile),
185+
WithEgressSelectionName(EgressSelectionNameEtcd),
186+
)
187+
Expect(err).To(HaveOccurred())
188+
})
125189
})
126190
})

go.mod

+35
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ require (
1313
k8s.io/api v0.24.3
1414
k8s.io/apiextensions-apiserver v0.24.3
1515
k8s.io/apimachinery v0.24.3
16+
k8s.io/apiserver v0.24.3
1617
k8s.io/client-go v0.24.3
1718
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
1819
sigs.k8s.io/controller-runtime v0.12.3
@@ -21,14 +22,21 @@ require (
2122
)
2223

2324
require (
25+
github.com/NYTimes/gziphandler v1.1.1 // indirect
2426
github.com/PuerkitoBio/purell v1.1.1 // indirect
2527
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
28+
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect
29+
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
2630
github.com/beorn7/perks v1.0.1 // indirect
31+
github.com/blang/semver/v4 v4.0.0 // indirect
2732
github.com/bmatcuk/doublestar/v4 v4.0.2 // indirect
2833
github.com/cespare/xxhash/v2 v2.1.2 // indirect
34+
github.com/coreos/go-semver v0.3.0 // indirect
35+
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
2936
github.com/davecgh/go-spew v1.1.1 // indirect
3037
github.com/emicklei/go-restful v2.9.5+incompatible // indirect
3138
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
39+
github.com/felixge/httpsnoop v1.0.1 // indirect
3240
github.com/fsnotify/fsnotify v1.5.1 // indirect
3341
github.com/go-errors/errors v1.0.1 // indirect
3442
github.com/go-logr/logr v1.2.3 // indirect
@@ -38,16 +46,21 @@ require (
3846
github.com/gogo/protobuf v1.3.2 // indirect
3947
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
4048
github.com/golang/protobuf v1.5.2 // indirect
49+
github.com/google/cel-go v0.10.1 // indirect
4150
github.com/google/gnostic v0.5.7-v3refs // indirect
4251
github.com/google/go-cmp v0.5.9 // indirect
4352
github.com/google/gofuzz v1.1.0 // indirect
4453
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
4554
github.com/google/uuid v1.1.2 // indirect
55+
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
56+
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
4657
github.com/imdario/mergo v0.3.12 // indirect
58+
github.com/inconshreveable/mousetrap v1.0.0 // indirect
4759
github.com/josharian/intern v1.0.0 // indirect
4860
github.com/json-iterator/go v1.1.12 // indirect
4961
github.com/mailru/easyjson v0.7.6 // indirect
5062
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
63+
github.com/mitchellh/mapstructure v1.4.1 // indirect
5164
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
5265
github.com/modern-go/reflect2 v1.0.2 // indirect
5366
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
@@ -58,9 +71,28 @@ require (
5871
github.com/prometheus/client_model v0.2.0 // indirect
5972
github.com/prometheus/common v0.32.1 // indirect
6073
github.com/prometheus/procfs v0.7.3 // indirect
74+
github.com/spf13/cobra v1.4.0 // indirect
75+
github.com/stoewer/go-strcase v1.2.0 // indirect
6176
github.com/stretchr/objx v0.5.0 // indirect
6277
github.com/xlab/treeprint v1.1.0 // indirect
78+
go.etcd.io/etcd/api/v3 v3.5.1 // indirect
79+
go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
80+
go.etcd.io/etcd/client/v3 v3.5.1 // indirect
81+
go.opentelemetry.io/contrib v0.20.0 // indirect
82+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
83+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
84+
go.opentelemetry.io/otel v0.20.0 // indirect
85+
go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
86+
go.opentelemetry.io/otel/metric v0.20.0 // indirect
87+
go.opentelemetry.io/otel/sdk v0.20.0 // indirect
88+
go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
89+
go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
90+
go.opentelemetry.io/otel/trace v0.20.0 // indirect
91+
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
6392
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
93+
go.uber.org/atomic v1.7.0 // indirect
94+
go.uber.org/multierr v1.6.0 // indirect
95+
go.uber.org/zap v1.19.1 // indirect
6496
golang.org/x/mod v0.6.0 // indirect
6597
golang.org/x/net v0.1.0 // indirect
6698
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
@@ -72,13 +104,16 @@ require (
72104
golang.org/x/tools v0.2.0 // indirect
73105
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
74106
google.golang.org/appengine v1.6.7 // indirect
107+
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
108+
google.golang.org/grpc v1.40.0 // indirect
75109
google.golang.org/protobuf v1.28.0 // indirect
76110
gopkg.in/inf.v0 v0.9.1 // indirect
77111
gopkg.in/yaml.v2 v2.4.0 // indirect
78112
gopkg.in/yaml.v3 v3.0.1 // indirect
79113
k8s.io/component-base v0.24.3 // indirect
80114
k8s.io/klog/v2 v2.60.1 // indirect
81115
k8s.io/kube-openapi v0.0.0-20220401212409-b28bf2818661 // indirect
116+
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 // indirect
82117
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
83118
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
84119
sigs.k8s.io/yaml v1.3.0 // indirect

0 commit comments

Comments
 (0)