Support for passing additional args to API server, controller manager…

…, and scheduler (#3162) * Support for passing additional args to API server, controller manager, and scheduler Signed-off-by: Waleed Malik <[email protected]> * Minor fixes Signed-off-by: Marko Mudrinić <[email protected]> * Fix logic for merging feature gates Signed-off-by: Marko Mudrinić <[email protected]> --------- Signed-off-by: Waleed Malik <[email protected]> Signed-off-by: Marko Mudrinić <[email protected]> Co-authored-by: Marko Mudrinić <[email protected]>
kubermatic · May 10, 2024 · e073c4b · e073c4b
1 parent 21ce2be
commit e073c4b
Show file tree

Hide file tree

Showing 11 changed files with 611 additions and 6 deletions.
diff --git a/docs/api_reference/v1beta2.en.md b/docs/api_reference/v1beta2.en.md
@@ -1,6 +1,6 @@
 +++
 title = "v1beta2 API Reference"
-date = 2024-03-12T21:50:36+02:00
+date = 2024-04-29T17:38:32+05:00
 weight = 11
 +++
 ## v1beta2
@@ -22,6 +22,8 @@ weight = 11
 * [ContainerdRegistry](#containerdregistry)
 * [ContainerdRegistryAuthConfig](#containerdregistryauthconfig)
 * [ContainerdTLSConfig](#containerdtlsconfig)
+* [ControlPlaneComponentConfig](#controlplanecomponentconfig)
+* [ControlPlaneComponents](#controlplanecomponents)
 * [ControlPlaneConfig](#controlplaneconfig)
 * [CoreDNS](#coredns)
 * [DNSConfig](#dnsconfig)
@@ -282,6 +284,29 @@ Configures containerd TLS for a registry
 
 [Back to Group](#v1beta2)
 
+### ControlPlaneComponentConfig
+
+
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| flags | Flags is a set of additional flags that will be passed to the control plane component. KubeOne internally configures some flags that are eseeential for the cluster to work. Those flags set by KubeOne will be merged with the ones specified in the configuration. In case of conflict the value provided by the user will be used. Usage of `feature-gates` is not allowed here, use `FeatureGates` field instead. IMPORTANT: Use of these flags is at the user's own risk, as KubeOne does not provide support for issues caused by invalid values and configurations. | map[string]string | false |
+| featureGates | FeatureGates is a map of additional feature gates that will be passed on to the control plane component. KubeOne internally configures some feature gates that are eseeential for the cluster to work. Those feature gates set by KubeOne will be merged with the ones specified in the configuration. In case of conflict the value provided by the user will be used. IMPORTANT: Use of these featureGates is at the user's own risk, as KubeOne does not provide support for issues caused by invalid values and configurations. | map[string]bool | false |
+
+[Back to Group](#v1beta2)
+
+### ControlPlaneComponents
+
+
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| controllerManager | ControllerManagerConfig configures the Kubernetes Controller Manager | *[ControlPlaneComponentConfig](#controlplanecomponentconfig) | false |
+| scheduler | Scheduler configures the Kubernetes Scheduler | *[ControlPlaneComponentConfig](#controlplanecomponentconfig) | false |
+| apiServer | APIServer configures the Kubernetes API Server | *[ControlPlaneComponentConfig](#controlplanecomponentconfig) | false |
+
+[Back to Group](#v1beta2)
+
 ### ControlPlaneConfig
 
 ControlPlaneConfig defines control plane nodes
@@ -528,6 +553,7 @@ KubeOneCluster is KubeOne Cluster API Schema
 | registryConfiguration | RegistryConfiguration configures how Docker images are pulled from an image registry | *[RegistryConfiguration](#registryconfiguration) | false |
 | loggingConfig | LoggingConfig configures the Kubelet's log rotation | [LoggingConfig](#loggingconfig) | false |
 | tlsCipherSuites | TLSCipherSuites allows to configure TLS cipher suites for different components. See https://pkg.go.dev/crypto/tls#pkg-constants for possible values. | [TLSCipherSuites](#tlsciphersuites) | true |
+| controlPlaneComponents | ControlPlaneComponents configures the Kubernetes control plane components | *[ControlPlaneComponents](#controlplanecomponents) | false |
 
 [Back to Group](#v1beta2)
 

diff --git a/pkg/apis/kubeone/config/config.go b/pkg/apis/kubeone/config/config.go
@@ -42,7 +42,9 @@ import (
 
 const (
 	// KubeOneClusterKind is kind of the KubeOneCluster object
-	KubeOneClusterKind = "KubeOneCluster"
+	KubeOneClusterKind                  = "KubeOneCluster"
+	controlPlaneComponentsWarning       = "Usage of the .controlPlaneComponents feature is at your own risk since options configured via this feature cannot properly be validated by KubeOne"
+	flagsAndFeatureGateOverridesWarning = "\t- %s only covers %s. Some features might also need additional configuration for other components."
 )
 
 var (
@@ -185,7 +187,7 @@ func DefaultedV1Beta1KubeOneCluster(versionedCluster *kubeonev1beta1.KubeOneClus
 	}
 
 	// Check for deprecated fields/features for a cluster
-	checkClusterFeatures(*internalCluster, logger)
+	checkClusterConfiguration(*internalCluster, logger)
 
 	return internalCluster, nil
 }
@@ -222,7 +224,7 @@ func DefaultedV1Beta2KubeOneCluster(versionedCluster *kubeonev1beta2.KubeOneClus
 	}
 
 	// Check for deprecated fields/features for a cluster
-	checkClusterFeatures(*internalCluster, logger)
+	checkClusterConfiguration(*internalCluster, logger)
 
 	return internalCluster, nil
 }
@@ -338,8 +340,8 @@ func isDir(dirname string) bool {
 	return statErr == nil && stat.Mode().IsDir()
 }
 
-// checkClusterFeatures checks clusters for usage of alpha and deprecated fields, flags etc. and print a warning if any are found
-func checkClusterFeatures(cluster kubeoneapi.KubeOneCluster, logger logrus.FieldLogger) {
+// checkClusterConfiguration checks clusters for usage of alpha, deprecated fields, flags, unrecommended features etc. and print a warning if any are found.
+func checkClusterConfiguration(cluster kubeoneapi.KubeOneCluster, logger logrus.FieldLogger) {
 	if cluster.Features.PodSecurityPolicy != nil && cluster.Features.PodSecurityPolicy.Enable {
 		logger.Warnf("PodSecurityPolicy is deprecated and will be removed with Kubernetes 1.25 release")
 	}
@@ -351,4 +353,28 @@ func checkClusterFeatures(cluster kubeoneapi.KubeOneCluster, logger logrus.Field
 	if cluster.CloudProvider.Vsphere != nil && !cluster.CloudProvider.External && len(cluster.CloudProvider.CSIConfig) > 0 {
 		logger.Warnf(".cloudProvider.csiConfig is provided, but is ignored when used with the in-tree cloud provider")
 	}
+
+	checkFlagsAndFeatureGateOverrides(cluster, logger)
+}
+
+func checkFlagsAndFeatureGateOverrides(cluster kubeoneapi.KubeOneCluster, logger logrus.FieldLogger) {
+	if cluster.ControlPlaneComponents != nil {
+		logger.Warn(controlPlaneComponentsWarning)
+
+		if cluster.ControlPlaneComponents.ControllerManager != nil {
+			if cluster.ControlPlaneComponents.ControllerManager.Flags != nil || cluster.ControlPlaneComponents.ControllerManager.FeatureGates != nil {
+				logger.Warnf(flagsAndFeatureGateOverridesWarning, ".controlPlaneComponents.controllerManager", "kube-controller-manager")
+			}
+		}
+		if cluster.ControlPlaneComponents.Scheduler != nil {
+			if cluster.ControlPlaneComponents.Scheduler.Flags != nil || cluster.ControlPlaneComponents.Scheduler.FeatureGates != nil {
+				logger.Warnf(flagsAndFeatureGateOverridesWarning, ".controlPlaneComponents.scheduler", "kube-scheduler")
+			}
+		}
+		if cluster.ControlPlaneComponents.APIServer != nil {
+			if cluster.ControlPlaneComponents.APIServer.Flags != nil || cluster.ControlPlaneComponents.APIServer.FeatureGates != nil {
+				logger.Warnf(flagsAndFeatureGateOverridesWarning, ".controlPlaneComponents.apiServer", "kube-apiserver")
+			}
+		}
+	}
 }
diff --git a/pkg/apis/kubeone/types.go b/pkg/apis/kubeone/types.go
@@ -93,6 +93,35 @@ type KubeOneCluster struct {
 	// TLSCipherSuites allows to configure TLS cipher suites for different components. See
 	// https://pkg.go.dev/crypto/tls#pkg-constants for possible values.
 	TLSCipherSuites TLSCipherSuites `json:"tlsCipherSuites"`
+
+	// ControlPlaneComponents configures the Kubernetes control plane components
+	ControlPlaneComponents *ControlPlaneComponents `json:"controlPlaneComponents,omitempty"`
+}
+
+type ControlPlaneComponents struct {
+	// ControllerManagerConfig configures the Kubernetes Controller Manager
+	ControllerManager *ControlPlaneComponentConfig `json:"controllerManager,omitempty"`
+
+	// Scheduler configures the Kubernetes Scheduler
+	Scheduler *ControlPlaneComponentConfig `json:"scheduler,omitempty"`
+
+	// APIServer configures the Kubernetes API Server
+	APIServer *ControlPlaneComponentConfig `json:"apiServer,omitempty"`
+}
+
+type ControlPlaneComponentConfig struct {
+	// Flags is a set of additional flags that will be passed to the control plane component.
+	// KubeOne internally configures some flags that are eseeential for the cluster to work. Those flags set by KubeOne
+	// will be merged with the ones specified in the configuration. In case of conflict the value provided by the user will be used.
+	// Usage of `feature-gates` is not allowed here, use `FeatureGates` field instead.
+	// IMPORTANT: Use of these flags is at the user's own risk, as KubeOne does not provide support for issues caused by invalid values and configurations.
+	Flags map[string]string `json:"flags,omitempty"`
+
+	// FeatureGates is a map of additional feature gates that will be passed on to the control plane component.
+	// KubeOne internally configures some feature gates that are eseeential for the cluster to work. Those feature gates set by KubeOne
+	// will be merged with the ones specified in the configuration. In case of conflict the value provided by the user will be used.
+	// IMPORTANT: Use of these featureGates is at the user's own risk, as KubeOne does not provide support for issues caused by invalid values and configurations.
+	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
 type TLSCipherSuites struct {

diff --git a/pkg/apis/kubeone/v1beta1/zz_generated.conversion.go b/pkg/apis/kubeone/v1beta1/zz_generated.conversion.go
diff --git a/pkg/apis/kubeone/v1beta2/types.go b/pkg/apis/kubeone/v1beta2/types.go
@@ -90,6 +90,35 @@ type KubeOneCluster struct {
 	// TLSCipherSuites allows to configure TLS cipher suites for different components. See
 	// https://pkg.go.dev/crypto/tls#pkg-constants for possible values.
 	TLSCipherSuites TLSCipherSuites `json:"tlsCipherSuites"`
+
+	// ControlPlaneComponents configures the Kubernetes control plane components
+	ControlPlaneComponents *ControlPlaneComponents `json:"controlPlaneComponents,omitempty"`
+}
+
+type ControlPlaneComponents struct {
+	// ControllerManagerConfig configures the Kubernetes Controller Manager
+	ControllerManager *ControlPlaneComponentConfig `json:"controllerManager,omitempty"`
+
+	// Scheduler configures the Kubernetes Scheduler
+	Scheduler *ControlPlaneComponentConfig `json:"scheduler,omitempty"`
+
+	// APIServer configures the Kubernetes API Server
+	APIServer *ControlPlaneComponentConfig `json:"apiServer,omitempty"`
+}
+
+type ControlPlaneComponentConfig struct {
+	// Flags is a set of additional flags that will be passed to the control plane component.
+	// KubeOne internally configures some flags that are eseeential for the cluster to work. Those flags set by KubeOne
+	// will be merged with the ones specified in the configuration. In case of conflict the value provided by the user will be used.
+	// Usage of `feature-gates` is not allowed here, use `FeatureGates` field instead.
+	// IMPORTANT: Use of these flags is at the user's own risk, as KubeOne does not provide support for issues caused by invalid values and configurations.
+	Flags map[string]string `json:"flags,omitempty"`
+
+	// FeatureGates is a map of additional feature gates that will be passed on to the control plane component.
+	// KubeOne internally configures some feature gates that are eseeential for the cluster to work. Those feature gates set by KubeOne
+	// will be merged with the ones specified in the configuration. In case of conflict the value provided by the user will be used.
+	// IMPORTANT: Use of these featureGates is at the user's own risk, as KubeOne does not provide support for issues caused by invalid values and configurations.
+	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
 type TLSCipherSuites struct {

diff --git a/pkg/apis/kubeone/v1beta2/zz_generated.conversion.go b/pkg/apis/kubeone/v1beta2/zz_generated.conversion.go