Skip to content

Commit bea50af

Browse files
huangchenzhaohuangchenzhao
authored
feat: add local mode for YurtHub to enhance operational efficiency of K8s cluster in user's IDC (openyurtio#2156) (openyurtio#2157)
1 parent 750f834 commit bea50af

File tree

9 files changed

+466
-147
lines changed

9 files changed

+466
-147
lines changed

Diff for: ‎cmd/yurthub/app/config/config.go

+54-28
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ import (
6363
type YurtHubConfiguration struct {
6464
LBMode string
6565
RemoteServers []*url.URL
66+
TenantKasService string // ip:port, used in local mode
6667
GCFrequency int
6768
NodeName string
6869
HeartbeatFailedRetry int
@@ -99,11 +100,12 @@ type YurtHubConfiguration struct {
99100
CoordinatorStorageAddr string // ip:port
100101
CoordinatorClient kubernetes.Interface
101102
LeaderElection componentbaseconfig.LeaderElectionConfiguration
103+
HostControlPlaneAddr string // ip:port
102104
}
103105

104106
// Complete converts *options.YurtHubOptions to *YurtHubConfiguration
105107
func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
106-
us, err := parseRemoteServers(options.ServerAddr)
108+
us, err := parseRemoteServers(options.WorkingMode, options.ServerAddr)
107109
if err != nil {
108110
return nil, err
109111
}
@@ -173,42 +175,53 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
173175
CoordinatorStoragePrefix: options.CoordinatorStoragePrefix,
174176
CoordinatorStorageAddr: options.CoordinatorStorageAddr,
175177
LeaderElection: options.LeaderElection,
178+
HostControlPlaneAddr: options.HostControlPlaneAddr,
176179
}
177180

178-
certMgr, err := certificatemgr.NewYurtHubCertManager(options, us)
179-
if err != nil {
180-
return nil, err
181-
}
182-
certMgr.Start()
183-
err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 4*time.Minute, true, func(ctx context.Context) (bool, error) {
184-
isReady := certMgr.Ready()
185-
if isReady {
186-
return true, nil
181+
// if yurthub is in local mode, certMgr and networkMgr are no need to start
182+
if cfg.WorkingMode != util.WorkingModeLocal {
183+
certMgr, err := certificatemgr.NewYurtHubCertManager(options, us)
184+
if err != nil {
185+
return nil, err
187186
}
188-
return false, nil
189-
})
190-
if err != nil {
191-
return nil, fmt.Errorf("hub certificates preparation failed, %v", err)
192-
}
193-
cfg.CertManager = certMgr
194-
195-
if options.EnableDummyIf {
196-
klog.V(2).Infof("create dummy network interface %s(%s) and init iptables manager", options.HubAgentDummyIfName, options.HubAgentDummyIfIP)
197-
networkMgr, err := network.NewNetworkManager(options)
187+
certMgr.Start()
188+
err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 4*time.Minute, true, func(ctx context.Context) (bool, error) {
189+
isReady := certMgr.Ready()
190+
if isReady {
191+
return true, nil
192+
}
193+
return false, nil
194+
})
198195
if err != nil {
199-
return nil, fmt.Errorf("could not create network manager, %w", err)
196+
return nil, fmt.Errorf("hub certificates preparation failed, %v", err)
200197
}
201-
cfg.NetworkMgr = networkMgr
202-
}
198+
cfg.CertManager = certMgr
203199

204-
if err = prepareServerServing(options, certMgr, cfg); err != nil {
205-
return nil, err
200+
if options.EnableDummyIf {
201+
klog.V(2).Infof("create dummy network interface %s(%s) and init iptables manager", options.HubAgentDummyIfName, options.HubAgentDummyIfIP)
202+
networkMgr, err := network.NewNetworkManager(options)
203+
if err != nil {
204+
return nil, fmt.Errorf("could not create network manager, %w", err)
205+
}
206+
cfg.NetworkMgr = networkMgr
207+
}
208+
209+
if err = prepareServerServing(options, certMgr, cfg); err != nil {
210+
return nil, err
211+
}
212+
} else {
213+
// if yurthub is in local mode, cfg.TenantKasService is used to represented as the service address (ip:port) of multiple apiserver daemonsets
214+
cfg.TenantKasService = options.ServerAddr
206215
}
207216

208217
return cfg, nil
209218
}
210219

211-
func parseRemoteServers(serverAddr string) ([]*url.URL, error) {
220+
func parseRemoteServers(workingMode string, serverAddr string) ([]*url.URL, error) {
221+
// if yurthub is in local mode, the format of serverAddr is ip:port, skip this function
222+
if workingMode == string(util.WorkingModeLocal) {
223+
return nil, nil
224+
}
212225
if serverAddr == "" {
213226
return make([]*url.URL, 0), fmt.Errorf("--server-addr should be set for hub agent")
214227
}
@@ -242,7 +255,12 @@ func parseRemoteServers(serverAddr string) ([]*url.URL, error) {
242255
func createClientAndSharedInformers(options *options.YurtHubOptions) (kubernetes.Interface, informers.SharedInformerFactory, dynamicinformer.DynamicSharedInformerFactory, error) {
243256
var kubeConfig *rest.Config
244257
var err error
245-
kubeConfig, err = clientcmd.BuildConfigFromFlags(fmt.Sprintf("http://%s:%d", options.YurtHubProxyHost, options.YurtHubProxyPort), "")
258+
// If yurthub is in local mode, create kubeconfig for host control plane to prepare informerFactory.
259+
if util.WorkingMode(options.WorkingMode) == util.WorkingModeLocal {
260+
kubeConfig, err = clientcmd.BuildConfigFromFlags(fmt.Sprintf("http://%s", options.HostControlPlaneAddr), "")
261+
} else {
262+
kubeConfig, err = clientcmd.BuildConfigFromFlags(fmt.Sprintf("http://%s:%d", options.YurtHubProxyHost, options.YurtHubProxyPort), "")
263+
}
246264
if err != nil {
247265
return nil, nil, nil, err
248266
}
@@ -273,7 +291,7 @@ func createClientAndSharedInformers(options *options.YurtHubOptions) (kubernetes
273291
return client, informers.NewSharedInformerFactory(client, 24*time.Hour), dynamicInformerFactory, nil
274292
}
275293

276-
// registerInformers reconstruct configmap/secret/pod informers
294+
// registerInformers reconstruct configmap/secret/pod/endpoints informers
277295
func registerInformers(options *options.YurtHubOptions,
278296
informerFactory informers.SharedInformerFactory,
279297
workingMode util.WorkingMode,
@@ -320,6 +338,14 @@ func registerInformers(options *options.YurtHubOptions,
320338
return informer
321339
}
322340
informerFactory.InformerFor(&corev1.Service{}, newServiceInformer)
341+
342+
// endpoints informer is used in local working mode
343+
if workingMode == util.WorkingModeLocal {
344+
newEndpointsInformer := func(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
345+
return coreinformers.NewFilteredEndpointsInformer(client, "kube-public", resyncPeriod, nil, nil)
346+
}
347+
informerFactory.InformerFor(&corev1.Endpoints{}, newEndpointsInformer)
348+
}
323349
}
324350

325351
func prepareServerServing(options *options.YurtHubOptions, certMgr certificate.YurtCertificateManager, cfg *YurtHubConfiguration) error {

Diff for: ‎cmd/yurthub/app/options/options.go

+28-20
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ type YurtHubOptions struct {
7272
EnableIptables bool
7373
HubAgentDummyIfIP string
7474
HubAgentDummyIfName string
75+
HostControlPlaneAddr string
7576
DiskCachePath string
7677
EnableResourceFilter bool
7778
DisabledResourceFilters []string
@@ -149,30 +150,36 @@ func (options *YurtHubOptions) Validate() error {
149150
return fmt.Errorf("server-address is empty")
150151
}
151152

152-
if options.BootstrapMode != certificate.KubeletCertificateBootstrapMode {
153-
if len(options.JoinToken) == 0 && len(options.BootstrapFile) == 0 {
154-
return fmt.Errorf("bootstrap token and bootstrap file are empty, one of them must be set")
153+
if options.WorkingMode != string(util.WorkingModeLocal) {
154+
if options.BootstrapMode != certificate.KubeletCertificateBootstrapMode {
155+
if len(options.JoinToken) == 0 && len(options.BootstrapFile) == 0 {
156+
return fmt.Errorf("bootstrap token and bootstrap file are empty, one of them must be set")
157+
}
155158
}
156-
}
157159

158-
if !util.IsSupportedLBMode(options.LBMode) {
159-
return fmt.Errorf("lb mode(%s) is not supported", options.LBMode)
160-
}
160+
if !util.IsSupportedLBMode(options.LBMode) {
161+
return fmt.Errorf("lb mode(%s) is not supported", options.LBMode)
162+
}
161163

162-
if !util.IsSupportedWorkingMode(util.WorkingMode(options.WorkingMode)) {
163-
return fmt.Errorf("working mode %s is not supported", options.WorkingMode)
164-
}
164+
if !util.IsSupportedWorkingMode(util.WorkingMode(options.WorkingMode)) {
165+
return fmt.Errorf("working mode %s is not supported", options.WorkingMode)
166+
}
165167

166-
if err := options.verifyDummyIP(); err != nil {
167-
return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err)
168-
}
168+
if err := options.verifyDummyIP(); err != nil {
169+
return fmt.Errorf("dummy ip %s is not invalid, %w", options.HubAgentDummyIfIP, err)
170+
}
169171

170-
if len(options.HubAgentDummyIfName) > 15 {
171-
return fmt.Errorf("dummy name %s length should not be more than 15", options.HubAgentDummyIfName)
172-
}
172+
if len(options.HubAgentDummyIfName) > 15 {
173+
return fmt.Errorf("dummy name %s length should not be more than 15", options.HubAgentDummyIfName)
174+
}
173175

174-
if len(options.CACertHashes) == 0 && !options.UnsafeSkipCAVerification {
175-
return fmt.Errorf("set --discovery-token-unsafe-skip-ca-verification flag as true or pass CACertHashes to continue")
176+
if len(options.CACertHashes) == 0 && !options.UnsafeSkipCAVerification {
177+
return fmt.Errorf("set --discovery-token-unsafe-skip-ca-verification flag as true or pass CACertHashes to continue")
178+
}
179+
} else {
180+
if len(options.HostControlPlaneAddr) == 0 {
181+
return fmt.Errorf("host-control-plane-address is empty")
182+
}
176183
}
177184

178185
return nil
@@ -186,7 +193,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
186193
fs.IntVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver")
187194
fs.IntVar(&o.YurtHubProxySecurePort, "proxy-secure-port", o.YurtHubProxySecurePort, "the port on which to proxy HTTPS requests to kube-apiserver")
188195
fs.StringVar(&o.YurtHubNamespace, "namespace", o.YurtHubNamespace, "the namespace of YurtHub Server")
189-
fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"")
196+
fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver, the format is: \"server1,server2,...\"; when yurthub is in local mode, server-addr represents the service address of apiservers, the format is: \"ip:port\".")
190197
fs.StringSliceVar(&o.YurtHubCertOrganizations, "hub-cert-organizations", o.YurtHubCertOrganizations, "Organizations that will be added into hub's apiserver client certificate, the format is: certOrg1,certOrg2,...")
191198
fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).")
192199
fs.StringVar(&o.NodeName, "node-name", o.NodeName, "the name of node that runs hub agent")
@@ -212,7 +219,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
212219
fs.BoolVar(&o.EnableResourceFilter, "enable-resource-filter", o.EnableResourceFilter, "enable to filter response that comes back from reverse proxy")
213220
fs.StringSliceVar(&o.DisabledResourceFilters, "disabled-resource-filters", o.DisabledResourceFilters, "disable resource filters to handle response")
214221
fs.StringVar(&o.NodePoolName, "nodepool-name", o.NodePoolName, "the name of node pool that runs hub agent")
215-
fs.StringVar(&o.WorkingMode, "working-mode", o.WorkingMode, "the working mode of yurthub(edge, cloud).")
222+
fs.StringVar(&o.WorkingMode, "working-mode", o.WorkingMode, "the working mode of yurthub(edge, cloud, local).")
216223
fs.DurationVar(&o.KubeletHealthGracePeriod, "kubelet-health-grace-period", o.KubeletHealthGracePeriod, "the amount of time which we allow kubelet to be unresponsive before stop renew node lease")
217224
fs.BoolVar(&o.EnableNodePool, "enable-node-pool", o.EnableNodePool, "enable list/watch nodepools resource or not for filters(only used for testing)")
218225
fs.MarkDeprecated("enable-node-pool", "It is planned to be removed from OpenYurt in the future version, please use --enable-pool-service-topology instead")
@@ -225,6 +232,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
225232
fs.StringVar(&o.CoordinatorStorageAddr, "coordinator-storage-addr", o.CoordinatorStorageAddr, "Address of Yurt-Coordinator etcd, in the format host:port")
226233
bindFlags(&o.LeaderElection, fs)
227234
fs.BoolVar(&o.EnablePoolServiceTopology, "enable-pool-service-topology", o.EnablePoolServiceTopology, "enable service topology feature in the node pool.")
235+
fs.StringVar(&o.HostControlPlaneAddr, "host-control-plane-address", o.HostControlPlaneAddr, "the address (ip:port) of host kubernetes cluster that used for yurthub local mode.")
228236
}
229237

230238
// bindFlags binds the LeaderElectionConfiguration struct fields to a flagset

0 commit comments

Comments
 (0)