From 66d8b5a459ac990bc249a1bdd565a03f7f58ceeb Mon Sep 17 00:00:00 2001 From: Yuxing Deng Date: Thu, 10 Oct 2024 13:44:24 +0800 Subject: [PATCH] feat: Use socket to do IPC request to kube-explorer --- Dockerfile | 6 +-- cmd/explorer.go | 36 +++++++++---- go.mod | 4 +- go.sum | 6 +-- hack/make-rules/autok3s.sh | 7 ++- pkg/cluster/cluster.go | 5 +- pkg/common/explorer.go | 84 ++++++++++++++---------------- pkg/common/explorer_unix.go | 29 +++++++++++ pkg/common/explorer_windows.go | 33 ++++++++++++ pkg/server/proxy/explorer.go | 62 ++++++++++++++-------- pkg/server/store/cluster/action.go | 5 +- 11 files changed, 179 insertions(+), 98 deletions(-) create mode 100644 pkg/common/explorer_unix.go create mode 100644 pkg/common/explorer_windows.go diff --git a/Dockerfile b/Dockerfile index d8006958..a20902ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ +ARG KUBE_EXPLORER_VERSION=latest +FROM cnrancher/kube-explorer:${KUBE_EXPLORER_VERSION} as kube-explorer FROM registry.suse.com/bci/bci-base:15.5 ARG TARGETPLATFORM ARG TARGETARCH ARG TARGETOS ENV TARGETPLATFORM=${TARGETPLATFORM:-"linux/amd64"} ARCH=${TARGETARCH:-"amd64"} OS=${TARGETOS:-"linux"} -ENV KUBE_EXPLORER_VERSION=v0.5.0 ENV HELM_DASHBOARD_VERSION=1.3.3 RUN zypper -n install curl ca-certificates tar gzip @@ -14,8 +15,7 @@ RUN mkdir /home/shell && \ echo 'source <(kubectl completion bash)' >> /home/shell/.bashrc && \ echo 'PS1="> "' >> /home/shell/.bashrc -RUN curl -sSL https://github.com/cnrancher/kube-explorer/releases/download/${KUBE_EXPLORER_VERSION}/kube-explorer-${OS}-${ARCH} > /usr/local/bin/kube-explorer && \ - chmod +x /usr/local/bin/kube-explorer +COPY --from=kube-explorer /usr/bin/kube-explorer /usr/local/bin/kube-explorer RUN curl -sLf https://github.com/komodorio/helm-dashboard/releases/download/v${HELM_DASHBOARD_VERSION}/helm-dashboard_${HELM_DASHBOARD_VERSION}_Linux_x86_64.tar.gz | tar xvzf - -C /usr/local/bin && \ chmod +x /usr/local/bin/helm-dashboard diff --git a/cmd/explorer.go b/cmd/explorer.go index b2c7fb12..e575b6c4 100644 --- a/cmd/explorer.go +++ b/cmd/explorer.go @@ -1,9 +1,14 @@ package cmd import ( + "context" + "fmt" + "net" + "net/http" + "github.com/cnrancher/autok3s/pkg/common" + "github.com/cnrancher/autok3s/pkg/server/proxy" - k3dutil "github.com/k3d-io/k3d/v5/cmd/util" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -15,7 +20,7 @@ var ( Example: "autok3s explorer --context myk3s", } clusterID = "" - explorerPort = 0 + explorerPort = 8080 ) func init() { @@ -31,18 +36,29 @@ func ExplorerCommand() *cobra.Command { } return nil } - explorerCmd.Run = func(_ *cobra.Command, _ []string) { + explorerCmd.Run = func(cmd *cobra.Command, _ []string) { if err := common.CheckCommandExist(common.KubeExplorerCommand); err != nil { logrus.Fatalln(err) } - if explorerPort == 0 { - port, err := k3dutil.GetFreePort() - if err != nil { - logrus.Fatalf("failed to get free port for kube-explorer: %v", err) - } - explorerPort = port + + wait, err := common.StartKubeExplorer(cmd.Context(), clusterID) + if err != nil { + logrus.Fatalln(err) + } + + server := http.Server{ + Addr: fmt.Sprintf(":%d", explorerPort), + Handler: proxy.DynamicPrefixProxy(clusterID), + BaseContext: func(_ net.Listener) context.Context { + return cmd.Context() + }, } - _ = common.StartKubeExplorer(explorerCmd.Context(), clusterID, explorerPort) + go func() { + logrus.Infof("autok3s serving kube-explorer on %s", server.Addr) + _ = server.ListenAndServe() + }() + <-wait + _ = server.Shutdown(context.Background()) } return explorerCmd diff --git a/go.mod b/go.mod index 1f0864c8..c7f80ec4 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( ) require ( + github.com/Microsoft/go-winio v0.6.2 github.com/moby/sys/signal v0.7.0 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 sigs.k8s.io/yaml v1.4.0 @@ -71,7 +72,6 @@ require ( github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -190,11 +190,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/mod v0.14.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/grpc v1.59.0 // indirect diff --git a/go.sum b/go.sum index 4a997940..fb7b84f1 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= @@ -594,8 +594,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/hack/make-rules/autok3s.sh b/hack/make-rules/autok3s.sh index f60cd14e..82852ba0 100755 --- a/hack/make-rules/autok3s.sh +++ b/hack/make-rules/autok3s.sh @@ -66,9 +66,8 @@ function build() { go_generate autok3s::log::info "building autok3s(${GIT_VERSION},${GIT_COMMIT},${GIT_TREE_STATE},${BUILD_DATE})..." # TODO default k3s version in k3d should also get from k3d in - local K3D_VERSION=`go list -m all | grep k3d/v5 | awk '{print $2}'` - local K3S_TAG=`curl --silent --retry 3 "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/'` - + local K3D_VERSION=$(go list -m github.com/k3d-io/k3d/v5 | awk '{print $2}') + local K3S_TAG=$(curl --silent --retry 3 'https://update.k3s.io/v1-release/channels/stable' | grep -E -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/') local version_flags=" -X main.gitVersion=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT} @@ -81,7 +80,7 @@ function build() { -X k8s.io/component-base/version.gitCommit=${GIT_COMMIT} -X k8s.io/component-base/version.gitTreeState=${GIT_TREE_STATE} -X k8s.io/component-base/version.buildDate=${BUILD_DATE} - -X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.4} + -X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.6} -X github.com/k3d-io/k3d/v5/version.K3sVersion=${K3S_TAG}" local flags=" -w -s" diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index c63b80e3..f679c261 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -156,13 +156,10 @@ func (p *ProviderBase) InitK3sCluster(cluster *types.Cluster, deployCCM func() [ for plugin := range enabledPlugins { if plugin == "explorer" { // start kube-explorer - port, err := common.EnableExplorer(context.Background(), cluster.ContextName) + err := common.EnableExplorer(context.Background(), cluster.ContextName) if err != nil { p.Logger.Errorf("[%s] failed to start kube-explorer for cluster %s: %v", p.Provider, cluster.ContextName, err) } - if port != 0 { - p.Logger.Infof("[%s] kube-explorer for cluster %s will listen on 127.0.0.1:%d...", p.Provider, cluster.Name, port) - } } } diff --git a/pkg/common/explorer.go b/pkg/common/explorer.go index f637cb35..37a4c4ef 100644 --- a/pkg/common/explorer.go +++ b/pkg/common/explorer.go @@ -6,8 +6,8 @@ import ( "os" "os/exec" "path/filepath" + "time" - k3dutil "github.com/k3d-io/k3d/v5/cmd/util" "github.com/sirupsen/logrus" ) @@ -16,77 +16,66 @@ const ( ) // EnableExplorer will start kube-explorer with random port for specified K3s cluster -func EnableExplorer(ctx context.Context, config string) (int, error) { +func EnableExplorer(ctx context.Context, config string) error { if _, ok := ExplorerWatchers[config]; ok { - return 0, fmt.Errorf("kube-explorer for cluster %s has already started", config) + return fmt.Errorf("kube-explorer for cluster %s has already started", config) } if err := CheckCommandExist(KubeExplorerCommand); err != nil { - return 0, err + return err } // command execution validate if err := checkExplorerCmd(); err != nil { - return 0, err + return err } // save config for kube-explorer exp, err := DefaultDB.GetExplorer(config) if err != nil { - return 0, err + return err } if exp == nil || !exp.Enabled { - var port int - if exp == nil { - port, err = k3dutil.GetFreePort() - if err != nil { - return 0, err - } - } else { - port = exp.Port - } exp = &Explorer{ ContextName: config, - Port: port, Enabled: true, } if err = DefaultDB.SaveExplorer(exp); err != nil { - return 0, err + return err } } // start kube-explorer explorerCtx, cancel := context.WithCancel(ctx) ExplorerWatchers[config] = cancel - go func(ctx context.Context, config string, port int) { - _ = StartKubeExplorer(ctx, config, port) - }(explorerCtx, config, exp.Port) - return exp.Port, nil + + if _, err := StartKubeExplorer(explorerCtx, config); err != nil { + return err + } + + return nil } // DisableExplorer will stop kube-explorer server for specified K3s cluster func DisableExplorer(config string) error { - if _, ok := ExplorerWatchers[config]; !ok { + cancelFunc, ok := ExplorerWatchers[config] + if !ok { return fmt.Errorf("cann't disable unactive kube-explorer for cluster %s", config) } + // update kube-explorer settings exp, err := DefaultDB.GetExplorer(config) if err != nil { return err } + if exp != nil && exp.Enabled { + // stop kube-explorer + cancelFunc() + delete(ExplorerWatchers, config) + } if exp == nil || exp.Enabled { - var port int - if exp == nil { - port, err = k3dutil.GetFreePort() - if err != nil { - return err - } - } else { - port = exp.Port - } err = DefaultDB.SaveExplorer(&Explorer{ ContextName: config, - Port: port, Enabled: false, }) if err != nil { @@ -94,9 +83,6 @@ func DisableExplorer(config string) error { } } - // stop kube-explorer - ExplorerWatchers[config]() - delete(ExplorerWatchers, config) return nil } @@ -110,26 +96,34 @@ func InitExplorer(ctx context.Context) { for _, exp := range expList { if exp.Enabled { logrus.Infof("start kube-explorer for cluster %s", exp.ContextName) - go func(ctx context.Context, name string) { - if _, err = EnableExplorer(ctx, name); err != nil { - logrus.Errorf("failed to start kube-explorer for cluster %s: %v", name, err) - } - }(ctx, exp.ContextName) + if err = EnableExplorer(ctx, exp.ContextName); err != nil { + logrus.Errorf("failed to start kube-explorer for cluster %s: %v", exp.ContextName, err) + } } } } // StartKubeExplorer start kube-explorer server listen on specified port -func StartKubeExplorer(ctx context.Context, config string, port int) error { +func StartKubeExplorer(ctx context.Context, clusterID string) (chan int, error) { + socketName := GetSocketName(clusterID) explorer := exec.CommandContext(ctx, KubeExplorerCommand, fmt.Sprintf("--kubeconfig=%s", filepath.Join(CfgPath, KubeCfgFile)), - fmt.Sprintf("--context=%s", config), fmt.Sprintf("--http-listen-port=%d", port), "--https-listen-port=0") + fmt.Sprintf("--context=%s", clusterID), fmt.Sprintf("--bind-address=%s", socketName)) explorer.Stdout = os.Stdout explorer.Stderr = os.Stderr + explorer.Cancel = func() error { + return explorer.Process.Signal(os.Interrupt) + } + explorer.WaitDelay = 10 * time.Second if err := explorer.Start(); err != nil { - logrus.Errorf("fail to start kube-explorer for cluster %s: %v", config, err) + logrus.Errorf("fail to start kube-explorer for cluster %s: %v", clusterID, err) } - logrus.Infof("kube-explorer for %s K3s cluster will listen on 127.0.0.1:%d ...", config, port) - return explorer.Wait() + logrus.Infof("kube-explorer for %s K3s cluster will listen on %s ...", clusterID, socketName) + stopChan := make(chan int) + go func() { + _ = explorer.Wait() + close(stopChan) + }() + return stopChan, nil } func CheckCommandExist(cmd string) error { diff --git a/pkg/common/explorer_unix.go b/pkg/common/explorer_unix.go new file mode 100644 index 00000000..8cf2b335 --- /dev/null +++ b/pkg/common/explorer_unix.go @@ -0,0 +1,29 @@ +//go:build unix +// +build unix + +package common + +import ( + "context" + "fmt" + "net" + "net/url" + "path/filepath" +) + +const ( + socketFileName = "socket.sock" +) + +func GetSocketName(clusterID string) string { + return fmt.Sprintf("unix://%s", filepath.Join(GetClusterContextPath(clusterID), socketFileName)) +} + +func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, addr string) (net.Conn, error) { + clusterID, _, _ := net.SplitHostPort(addr) + u, _ := url.Parse(GetSocketName(clusterID)) + var d net.Dialer + return d.DialContext(ctx, "unix", u.Path) + } +} diff --git a/pkg/common/explorer_windows.go b/pkg/common/explorer_windows.go new file mode 100644 index 00000000..068af133 --- /dev/null +++ b/pkg/common/explorer_windows.go @@ -0,0 +1,33 @@ +//go:build windows +// +build windows + +package common + +import ( + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "net" + "net/url" + + "github.com/Microsoft/go-winio" +) + +func GetSocketName(clusterID string) string { + return fmt.Sprintf("namedpipe:/\\.\\pipe\\autok3s-%s", md5hash(clusterID)) +} + +func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, _, addr string) (net.Conn, error) { + clusterID, _, _ := net.SplitHostPort(addr) + u, _ := url.Parse(GetSocketName(clusterID)) + return winio.DialPipeContext(ctx, u.Path) + } +} + +func md5hash(s string) string { + hash := md5.Sum([]byte(s)) + hexStr := hex.EncodeToString(hash[:]) + return hexStr[:16] +} diff --git a/pkg/server/proxy/explorer.go b/pkg/server/proxy/explorer.go index b8daedae..bec5c0bf 100644 --- a/pkg/server/proxy/explorer.go +++ b/pkg/server/proxy/explorer.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "net/http/httputil" - "net/url" "strings" "github.com/cnrancher/autok3s/pkg/common" @@ -13,11 +12,14 @@ import ( ) type ExplorerHandler struct { + next http.Handler } // NewExplorerProxy return proxy handler for kube-explorer func NewExplorerProxy() http.Handler { - return &ExplorerHandler{} + return &ExplorerHandler{ + next: DynamicPrefixProxy(""), + } } // ServeHTTP handles the proxy request for kube-explorer @@ -43,27 +45,41 @@ func (ep *ExplorerHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) return } - u, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d/", explorer.Port)) - if err != nil { - rw.WriteHeader(http.StatusInternalServerError) - _, _ = rw.Write([]byte(err.Error())) - return - } - prefix := fmt.Sprintf("/proxy/explorer/%s", clusterID) + ep.next.ServeHTTP(rw, req) +} - proxy := &httputil.ReverseProxy{} - proxy.Director = func(req *http.Request) { - scheme := urlbuilder.GetScheme(req) - host := urlbuilder.GetHost(req, scheme) - req.Header.Set(urlbuilder.ForwardedProtoHeader, scheme) - req.Header.Set(urlbuilder.ForwardedHostHeader, host) - req.Header.Set(urlbuilder.PrefixHeader, prefix) - req.URL.Scheme = u.Scheme - req.URL.Host = u.Host - req.URL.Path = strings.TrimPrefix(req.URL.Path, prefix) - if req.URL.Path == "" { - req.URL.Path = "/" - } +func DynamicPrefixProxy(staticClusterID string) http.Handler { + proxy := &httputil.ReverseProxy{ + Transport: &http.Transport{ + DialContext: common.GetSocketDialer(), + }, + Director: func(req *http.Request) { + clusterID := staticClusterID + if clusterID == "" { + vars := mux.Vars(req) + clusterID = vars["name"] + } + // prefix only used for non static cluster proxy + var prefix string + if staticClusterID == "" { + prefix = fmt.Sprintf("/proxy/explorer/%s", clusterID) + } + scheme := urlbuilder.GetScheme(req) + host := urlbuilder.GetHost(req, scheme) + req.Header.Set(urlbuilder.ForwardedProtoHeader, scheme) + req.Header.Set(urlbuilder.ForwardedHostHeader, host) + req.URL.Scheme = scheme + req.URL.Host = clusterID + + if prefix != "" { + req.Header.Set(urlbuilder.PrefixHeader, prefix) + req.URL.Path = strings.TrimPrefix(req.URL.Path, prefix) + if req.URL.Path == "" { + req.URL.Path = "/" + } + } + }, } - proxy.ServeHTTP(rw, req) + + return proxy } diff --git a/pkg/server/store/cluster/action.go b/pkg/server/store/cluster/action.go index 3d78bf2e..f183fe9f 100644 --- a/pkg/server/store/cluster/action.go +++ b/pkg/server/store/cluster/action.go @@ -173,15 +173,16 @@ func (e explorer) ServeHTTP(_ http.ResponseWriter, req *http.Request) { action := apiRequest.Action switch action { case actionEnableExplorer: - port, err := common.EnableExplorer(context.Background(), clusterID) + err := common.EnableExplorer(context.Background(), clusterID) if err != nil { apiRequest.WriteError(apierror.NewAPIError(validation.ServerError, err.Error())) return } + socketName := common.GetSocketName(clusterID) apiRequest.WriteResponse(http.StatusOK, types.APIObject{ Type: "enableExplorerOutput", Object: &autok3stypes.EnableExplorerOutput{ - Data: fmt.Sprintf("kube-explorer for cluster %s will listen on 127.0.0.1:%d...", clusterID, port), + Data: fmt.Sprintf("kube-explorer for cluster %s will listen on %s ...", clusterID, socketName), }, }) case actionDisableExplorer: