From 79017208c51eea186d919c6f69b6273d85b975f1 Mon Sep 17 00:00:00 2001 From: Yuxing Deng Date: Sat, 12 Oct 2024 21:36:26 +0800 Subject: [PATCH] feat(k3d): Auto detect ip when setting 0.0.0.0 for API Port field When running autok3s in docker, 0.0.0.0 won't work as bridge network is used. Now the autok3s will detect docker host IP and set it to tls-san for k3d cluster Also when creating cluster via UI, the request hostname will also added to tls-san when running inside container. --- pkg/providers/k3d/docker.go | 135 +++++++++++++++++++++++++++++ pkg/providers/k3d/k3d.go | 34 +++++--- pkg/server/store/cluster/action.go | 12 +++ pkg/server/store/cluster/store.go | 10 ++- 4 files changed, 178 insertions(+), 13 deletions(-) create mode 100644 pkg/providers/k3d/docker.go diff --git a/pkg/providers/k3d/docker.go b/pkg/providers/k3d/docker.go new file mode 100644 index 00000000..296b9700 --- /dev/null +++ b/pkg/providers/k3d/docker.go @@ -0,0 +1,135 @@ +package k3d + +import ( + "context" + "fmt" + "net" + "net/url" + "os" + "strings" + "sync" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/k3d-io/k3d/v5/pkg/runtimes" + "github.com/k3d-io/k3d/v5/pkg/runtimes/docker" + "github.com/sirupsen/logrus" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +var ( + dockerHost string + once sync.Once +) + +func getDockerHost() string { + once.Do(func() { + var err error + if runtimes.SelectedRuntime.ID() != "docker" { + logrus.Debugf("runtime not docker") + return + } + // TODO find a better way to get container id via docker.sock + // Hostname as container id is not 100% reliable. + hostname := os.Getenv("HOSTNAME_OVERRIDE") + if hostname == "" { + hostname, err = os.Hostname() + if err != nil { + logrus.Debugf("failed to get hostname, %v", err) + return + } + } + + runtime := runtimes.Docker + + if runtime.GetHost() != "" { + logrus.Debugf("runtime host is %s", runtime.GetHost()) + return + } + + if _, err = os.Stat(runtime.GetRuntimePath()); err != nil { + logrus.Debugf("runtime path %s doesn't exist", runtime.GetRuntimePath()) + return + } + nodes, err := getContainersByLabel(context.Background(), map[string]string{ + "org.opencontainers.image.title": "autok3s", + }) + if err != nil { + logrus.Debugf("failed to get container from runtime %s, %v", runtime.ID(), err) + return + } + if len(nodes) == 0 { + logrus.Debug("autok3s docker container not found. Skip finding docker host IP.") + return + } + var currentContainer *types.Container + for i := range nodes { + node := nodes[i] + if strings.HasPrefix(node.ID, hostname) { + currentContainer = &node + break + } + } + if currentContainer == nil { + logrus.Debugf("no container found for hostname %s", hostname) + return + } + if currentContainer.HostConfig.NetworkMode == "host" { + logrus.Debug("do nothing when running host network") + return + } + logrus.Debugf("found container %s", currentContainer.ID) + gw, err := runtime.GetHostIP(context.Background(), currentContainer.HostConfig.NetworkMode) + if err != nil { + logrus.Debugf("failed to get gateway ip for network %s, %v", currentContainer.HostConfig.NetworkMode, err) + return + } + dockerHost = gw.String() + logrus.Infof("found docker host IP %s", dockerHost) + }) + return dockerHost +} + +func getContainersByLabel(ctx context.Context, labels map[string]string) ([]types.Container, error) { + // (0) create docker client + docker, err := docker.GetDockerClient() + if err != nil { + return nil, fmt.Errorf("Failed to create docker client. %+v", err) + } + defer docker.Close() + + filters := filters.NewArgs() + for k, v := range labels { + filters.Add("label", fmt.Sprintf("%s=%s", k, v)) + } + + containers, err := docker.ContainerList(ctx, container.ListOptions{ + Filters: filters, + All: true, + }) + if err != nil { + return nil, fmt.Errorf("failed to list containers: %w", err) + } + + return containers, nil +} + +func OverrideK3dKubeConfigServer(from, to string, config *clientcmdapi.Config) { + if to == "" { + return + } + if from == "" && getDockerHost() != "" { + from = getDockerHost() + } + for key := range config.Clusters { + cluster := config.Clusters[key] + serverURL, _ := url.Parse(cluster.Server) + if serverURL.Hostname() == from { + _, port, _ := net.SplitHostPort(serverURL.Host) + serverURL.Host = fmt.Sprintf("%s:%s", to, port) + } + cluster.Server = serverURL.String() + return + } +} diff --git a/pkg/providers/k3d/k3d.go b/pkg/providers/k3d/k3d.go index e3596502..ca6c972f 100644 --- a/pkg/providers/k3d/k3d.go +++ b/pkg/providers/k3d/k3d.go @@ -45,6 +45,8 @@ var ( type K3d struct { *cluster.ProviderBase `json:",inline"` typesk3d.Options `json:",inline"` + + dockerHost, additionalHost string } func init() { @@ -62,6 +64,7 @@ func newProvider() *K3d { APIPort: k3dAPIPort, Image: k3dImage, }, + dockerHost: getDockerHost(), } } @@ -173,6 +176,11 @@ func (p *K3d) GetProviderOptions(opt []byte) (interface{}, error) { // SetConfig set cluster config. func (p *K3d) SetConfig(config []byte) error { + tmpHost := struct { + AdditionalHost string `json:"additionalHost,omitempty"` + }{} + _ = json.Unmarshal(config, &tmpHost) + c, err := p.SetClusterConfig(config) if err != nil { return err @@ -189,7 +197,7 @@ func (p *K3d) SetConfig(config []byte) error { } targetOption := reflect.ValueOf(opt).Elem() utils.MergeConfig(sourceOption, targetOption) - + p.additionalHost = tmpHost.AdditionalHost return nil } @@ -326,6 +334,8 @@ func (p *K3d) obtainKubeCfg() (kubeCfg, ip string, err error) { return } + OverrideK3dKubeConfigServer(k3d.DefaultAPIHost, p.dockerHost, kubeConfig) + bytes, err := clientcmd.Write(*kubeConfig) if err != nil { return @@ -611,20 +621,15 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error) } if p.APIPort != "" { - exposeAPI, err := k3dutil.ParsePortExposureSpec(p.APIPort, k3d.DefaultAPIPort) + apiPort := p.APIPort + if strings.HasSuffix(apiPort, ":0") { + apiPort = strings.TrimSuffix(apiPort, ":0") + ":random" + } + exposeAPI, err := k3dutil.ParsePortExposureSpec(apiPort, k3d.DefaultAPIPort) if err != nil { return nil, fmt.Errorf("[%s] cluster %s parse port config failed: %w", p.GetProviderName(), p.Name, err) } - cfg.ExposeAPI.HostIP = exposeAPI.Binding.HostIP - - if exposeAPI.Binding.HostPort == "0" { - exposeAPI, err = k3dutil.ParsePortExposureSpec("random", k3d.DefaultAPIPort) - if err != nil { - return nil, fmt.Errorf("[%s] cluster %s parse random port config failed: %w", p.GetProviderName(), p.Name, err) - } - } - cfg.ExposeAPI.HostPort = exposeAPI.Binding.HostPort p.APIPort = fmt.Sprintf("%s:%s", cfg.ExposeAPI.HostIP, cfg.ExposeAPI.HostPort) } @@ -645,6 +650,13 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error) cfg.Options.Runtime.AgentsMemory = p.WorkersMemory } + for _, host := range []string{p.additionalHost, p.dockerHost} { + if host == "" { + continue + } + p.MasterExtraArgs = strings.TrimPrefix(p.MasterExtraArgs+" --tls-san="+host, " ") + } + if p.MasterExtraArgs != "" { cfg.Options.K3sOptions.ExtraArgs = []k3dconf.K3sArgWithNodeFilters{} for _, arg := range strings.Split(p.MasterExtraArgs, " ") { diff --git a/pkg/server/store/cluster/action.go b/pkg/server/store/cluster/action.go index f183fe9f..9d1d86a0 100644 --- a/pkg/server/store/cluster/action.go +++ b/pkg/server/store/cluster/action.go @@ -5,16 +5,20 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "path/filepath" + "strings" "github.com/cnrancher/autok3s/pkg/common" "github.com/cnrancher/autok3s/pkg/providers" + "github.com/cnrancher/autok3s/pkg/providers/k3d" autok3stypes "github.com/cnrancher/autok3s/pkg/types/apis" "github.com/gorilla/mux" "github.com/rancher/apiserver/pkg/apierror" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/apiserver/pkg/urlbuilder" "github.com/rancher/wrangler/v2/pkg/schemas/validation" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" @@ -256,6 +260,14 @@ func (d downloadKubeconfig) ServeHTTP(_ http.ResponseWriter, req *http.Request) } } + if strings.HasPrefix(clusterID, "k3d-") { + host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req, "")) + // When parsing empty string as from parameter, the dockerHost will be used as the origin server + // if the dockerHost is empty(e.g. DOCKER_HOST is set), this function will do nothing as the k3d already use the + // proper cluster server address in kubeconfig + k3d.OverrideK3dKubeConfigServer("", host, ¤tCfg) + } + result, err := clientcmd.Write(currentCfg) if err != nil { apiRequest.WriteError(apierror.NewAPIError(validation.ServerError, err.Error())) diff --git a/pkg/server/store/cluster/store.go b/pkg/server/store/cluster/store.go index ee4ff5ab..881d14c1 100644 --- a/pkg/server/store/cluster/store.go +++ b/pkg/server/store/cluster/store.go @@ -3,6 +3,7 @@ package cluster import ( "encoding/json" "fmt" + "net" "strings" "github.com/cnrancher/autok3s/pkg/cluster" @@ -14,6 +15,7 @@ import ( "github.com/rancher/apiserver/pkg/apierror" "github.com/rancher/apiserver/pkg/store/empty" "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/apiserver/pkg/urlbuilder" "github.com/rancher/wrangler/v2/pkg/schemas/validation" "github.com/sirupsen/logrus" ) @@ -24,9 +26,13 @@ type Store struct { } // Create creates cluster based on the request data. -func (c *Store) Create(_ *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) { +func (c *Store) Create(req *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) { providerName := data.Data().String("provider") - b, err := json.Marshal(data.Data()) + // for k3d to add the request host for additional tls-san + objMap := data.Data() + host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req.Request, "")) + objMap.Set("additionalHost", host) + b, err := json.Marshal(objMap) if err != nil { return types.APIObject{}, err }