Skip to content

Commit

Permalink
feat(k3d): Auto detect ip when setting 0.0.0.0 for API Port field
Browse files Browse the repository at this point in the history
When running autok3s in docker, 0.0.0.0 won't work as bridge network is used.
Now the autok3s will detect docker host IP and set it to tls-san for k3d cluster
Also when creating cluster via UI, the request hostname will also added to tls-san when running inside container.
  • Loading branch information
orangedeng committed Oct 12, 2024
1 parent 66d8b5a commit 7901720
Show file tree
Hide file tree
Showing 4 changed files with 178 additions and 13 deletions.
135 changes: 135 additions & 0 deletions pkg/providers/k3d/docker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
package k3d

import (
"context"
"fmt"
"net"
"net/url"
"os"
"strings"
"sync"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/k3d-io/k3d/v5/pkg/runtimes"
"github.com/k3d-io/k3d/v5/pkg/runtimes/docker"
"github.com/sirupsen/logrus"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

var (
dockerHost string
once sync.Once
)

func getDockerHost() string {
once.Do(func() {
var err error
if runtimes.SelectedRuntime.ID() != "docker" {
logrus.Debugf("runtime not docker")
return
}
// TODO find a better way to get container id via docker.sock
// Hostname as container id is not 100% reliable.
hostname := os.Getenv("HOSTNAME_OVERRIDE")
if hostname == "" {
hostname, err = os.Hostname()
if err != nil {
logrus.Debugf("failed to get hostname, %v", err)
return
}
}

runtime := runtimes.Docker

if runtime.GetHost() != "" {
logrus.Debugf("runtime host is %s", runtime.GetHost())
return
}

if _, err = os.Stat(runtime.GetRuntimePath()); err != nil {
logrus.Debugf("runtime path %s doesn't exist", runtime.GetRuntimePath())
return
}
nodes, err := getContainersByLabel(context.Background(), map[string]string{
"org.opencontainers.image.title": "autok3s",
})
if err != nil {
logrus.Debugf("failed to get container from runtime %s, %v", runtime.ID(), err)
return
}
if len(nodes) == 0 {
logrus.Debug("autok3s docker container not found. Skip finding docker host IP.")
return
}
var currentContainer *types.Container
for i := range nodes {
node := nodes[i]
if strings.HasPrefix(node.ID, hostname) {
currentContainer = &node
break
}
}
if currentContainer == nil {
logrus.Debugf("no container found for hostname %s", hostname)
return
}
if currentContainer.HostConfig.NetworkMode == "host" {
logrus.Debug("do nothing when running host network")
return
}
logrus.Debugf("found container %s", currentContainer.ID)
gw, err := runtime.GetHostIP(context.Background(), currentContainer.HostConfig.NetworkMode)
if err != nil {
logrus.Debugf("failed to get gateway ip for network %s, %v", currentContainer.HostConfig.NetworkMode, err)
return
}
dockerHost = gw.String()
logrus.Infof("found docker host IP %s", dockerHost)
})
return dockerHost
}

func getContainersByLabel(ctx context.Context, labels map[string]string) ([]types.Container, error) {
// (0) create docker client
docker, err := docker.GetDockerClient()
if err != nil {
return nil, fmt.Errorf("Failed to create docker client. %+v", err)
}
defer docker.Close()

filters := filters.NewArgs()
for k, v := range labels {
filters.Add("label", fmt.Sprintf("%s=%s", k, v))
}

containers, err := docker.ContainerList(ctx, container.ListOptions{
Filters: filters,
All: true,
})
if err != nil {
return nil, fmt.Errorf("failed to list containers: %w", err)
}

return containers, nil
}

func OverrideK3dKubeConfigServer(from, to string, config *clientcmdapi.Config) {
if to == "" {
return
}
if from == "" && getDockerHost() != "" {
from = getDockerHost()
}
for key := range config.Clusters {
cluster := config.Clusters[key]
serverURL, _ := url.Parse(cluster.Server)
if serverURL.Hostname() == from {
_, port, _ := net.SplitHostPort(serverURL.Host)
serverURL.Host = fmt.Sprintf("%s:%s", to, port)
}
cluster.Server = serverURL.String()
return
}
}
34 changes: 23 additions & 11 deletions pkg/providers/k3d/k3d.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ var (
type K3d struct {
*cluster.ProviderBase `json:",inline"`
typesk3d.Options `json:",inline"`

dockerHost, additionalHost string
}

func init() {
Expand All @@ -62,6 +64,7 @@ func newProvider() *K3d {
APIPort: k3dAPIPort,
Image: k3dImage,
},
dockerHost: getDockerHost(),
}
}

Expand Down Expand Up @@ -173,6 +176,11 @@ func (p *K3d) GetProviderOptions(opt []byte) (interface{}, error) {

// SetConfig set cluster config.
func (p *K3d) SetConfig(config []byte) error {
tmpHost := struct {
AdditionalHost string `json:"additionalHost,omitempty"`
}{}
_ = json.Unmarshal(config, &tmpHost)

c, err := p.SetClusterConfig(config)
if err != nil {
return err
Expand All @@ -189,7 +197,7 @@ func (p *K3d) SetConfig(config []byte) error {
}
targetOption := reflect.ValueOf(opt).Elem()
utils.MergeConfig(sourceOption, targetOption)

p.additionalHost = tmpHost.AdditionalHost
return nil
}

Expand Down Expand Up @@ -326,6 +334,8 @@ func (p *K3d) obtainKubeCfg() (kubeCfg, ip string, err error) {
return
}

OverrideK3dKubeConfigServer(k3d.DefaultAPIHost, p.dockerHost, kubeConfig)

bytes, err := clientcmd.Write(*kubeConfig)
if err != nil {
return
Expand Down Expand Up @@ -611,20 +621,15 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error)
}

if p.APIPort != "" {
exposeAPI, err := k3dutil.ParsePortExposureSpec(p.APIPort, k3d.DefaultAPIPort)
apiPort := p.APIPort
if strings.HasSuffix(apiPort, ":0") {
apiPort = strings.TrimSuffix(apiPort, ":0") + ":random"
}
exposeAPI, err := k3dutil.ParsePortExposureSpec(apiPort, k3d.DefaultAPIPort)
if err != nil {
return nil, fmt.Errorf("[%s] cluster %s parse port config failed: %w", p.GetProviderName(), p.Name, err)
}

cfg.ExposeAPI.HostIP = exposeAPI.Binding.HostIP

if exposeAPI.Binding.HostPort == "0" {
exposeAPI, err = k3dutil.ParsePortExposureSpec("random", k3d.DefaultAPIPort)
if err != nil {
return nil, fmt.Errorf("[%s] cluster %s parse random port config failed: %w", p.GetProviderName(), p.Name, err)
}
}

cfg.ExposeAPI.HostPort = exposeAPI.Binding.HostPort
p.APIPort = fmt.Sprintf("%s:%s", cfg.ExposeAPI.HostIP, cfg.ExposeAPI.HostPort)
}
Expand All @@ -645,6 +650,13 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error)
cfg.Options.Runtime.AgentsMemory = p.WorkersMemory
}

for _, host := range []string{p.additionalHost, p.dockerHost} {
if host == "" {
continue
}
p.MasterExtraArgs = strings.TrimPrefix(p.MasterExtraArgs+" --tls-san="+host, " ")
}

if p.MasterExtraArgs != "" {
cfg.Options.K3sOptions.ExtraArgs = []k3dconf.K3sArgWithNodeFilters{}
for _, arg := range strings.Split(p.MasterExtraArgs, " ") {
Expand Down
12 changes: 12 additions & 0 deletions pkg/server/store/cluster/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,20 @@ import (
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"path/filepath"
"strings"

"github.com/cnrancher/autok3s/pkg/common"
"github.com/cnrancher/autok3s/pkg/providers"
"github.com/cnrancher/autok3s/pkg/providers/k3d"
autok3stypes "github.com/cnrancher/autok3s/pkg/types/apis"

"github.com/gorilla/mux"
"github.com/rancher/apiserver/pkg/apierror"
"github.com/rancher/apiserver/pkg/types"
"github.com/rancher/apiserver/pkg/urlbuilder"
"github.com/rancher/wrangler/v2/pkg/schemas/validation"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -256,6 +260,14 @@ func (d downloadKubeconfig) ServeHTTP(_ http.ResponseWriter, req *http.Request)
}
}

if strings.HasPrefix(clusterID, "k3d-") {
host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req, ""))
// When parsing empty string as from parameter, the dockerHost will be used as the origin server
// if the dockerHost is empty(e.g. DOCKER_HOST is set), this function will do nothing as the k3d already use the
// proper cluster server address in kubeconfig
k3d.OverrideK3dKubeConfigServer("", host, &currentCfg)
}

result, err := clientcmd.Write(currentCfg)
if err != nil {
apiRequest.WriteError(apierror.NewAPIError(validation.ServerError, err.Error()))
Expand Down
10 changes: 8 additions & 2 deletions pkg/server/store/cluster/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cluster
import (
"encoding/json"
"fmt"
"net"
"strings"

"github.com/cnrancher/autok3s/pkg/cluster"
Expand All @@ -14,6 +15,7 @@ import (
"github.com/rancher/apiserver/pkg/apierror"
"github.com/rancher/apiserver/pkg/store/empty"
"github.com/rancher/apiserver/pkg/types"
"github.com/rancher/apiserver/pkg/urlbuilder"
"github.com/rancher/wrangler/v2/pkg/schemas/validation"
"github.com/sirupsen/logrus"
)
Expand All @@ -24,9 +26,13 @@ type Store struct {
}

// Create creates cluster based on the request data.
func (c *Store) Create(_ *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) {
func (c *Store) Create(req *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) {
providerName := data.Data().String("provider")
b, err := json.Marshal(data.Data())
// for k3d to add the request host for additional tls-san
objMap := data.Data()
host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req.Request, ""))
objMap.Set("additionalHost", host)
b, err := json.Marshal(objMap)
if err != nil {
return types.APIObject{}, err
}
Expand Down

0 comments on commit 7901720

Please sign in to comment.