Skip to content

Commit

Permalink
feat: Use socket to do IPC request to kube-explorer
Browse files Browse the repository at this point in the history
  • Loading branch information
orangedeng committed Oct 12, 2024
1 parent 99f81b4 commit 66d8b5a
Show file tree
Hide file tree
Showing 11 changed files with 179 additions and 98 deletions.
6 changes: 3 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
ARG KUBE_EXPLORER_VERSION=latest
FROM cnrancher/kube-explorer:${KUBE_EXPLORER_VERSION} as kube-explorer
FROM registry.suse.com/bci/bci-base:15.5
ARG TARGETPLATFORM
ARG TARGETARCH
ARG TARGETOS

ENV TARGETPLATFORM=${TARGETPLATFORM:-"linux/amd64"} ARCH=${TARGETARCH:-"amd64"} OS=${TARGETOS:-"linux"}
ENV KUBE_EXPLORER_VERSION=v0.5.0
ENV HELM_DASHBOARD_VERSION=1.3.3

RUN zypper -n install curl ca-certificates tar gzip
Expand All @@ -14,8 +15,7 @@ RUN mkdir /home/shell && \
echo 'source <(kubectl completion bash)' >> /home/shell/.bashrc && \
echo 'PS1="> "' >> /home/shell/.bashrc

RUN curl -sSL https://github.com/cnrancher/kube-explorer/releases/download/${KUBE_EXPLORER_VERSION}/kube-explorer-${OS}-${ARCH} > /usr/local/bin/kube-explorer && \
chmod +x /usr/local/bin/kube-explorer
COPY --from=kube-explorer /usr/bin/kube-explorer /usr/local/bin/kube-explorer

RUN curl -sLf https://github.com/komodorio/helm-dashboard/releases/download/v${HELM_DASHBOARD_VERSION}/helm-dashboard_${HELM_DASHBOARD_VERSION}_Linux_x86_64.tar.gz | tar xvzf - -C /usr/local/bin && \
chmod +x /usr/local/bin/helm-dashboard
Expand Down
36 changes: 26 additions & 10 deletions cmd/explorer.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
package cmd

import (
"context"
"fmt"
"net"
"net/http"

"github.com/cnrancher/autok3s/pkg/common"
"github.com/cnrancher/autok3s/pkg/server/proxy"

k3dutil "github.com/k3d-io/k3d/v5/cmd/util"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand All @@ -15,7 +20,7 @@ var (
Example: "autok3s explorer --context myk3s",
}
clusterID = ""
explorerPort = 0
explorerPort = 8080
)

func init() {
Expand All @@ -31,18 +36,29 @@ func ExplorerCommand() *cobra.Command {
}
return nil
}
explorerCmd.Run = func(_ *cobra.Command, _ []string) {
explorerCmd.Run = func(cmd *cobra.Command, _ []string) {
if err := common.CheckCommandExist(common.KubeExplorerCommand); err != nil {
logrus.Fatalln(err)
}
if explorerPort == 0 {
port, err := k3dutil.GetFreePort()
if err != nil {
logrus.Fatalf("failed to get free port for kube-explorer: %v", err)
}
explorerPort = port

wait, err := common.StartKubeExplorer(cmd.Context(), clusterID)
if err != nil {
logrus.Fatalln(err)
}

server := http.Server{
Addr: fmt.Sprintf(":%d", explorerPort),
Handler: proxy.DynamicPrefixProxy(clusterID),
BaseContext: func(_ net.Listener) context.Context {
return cmd.Context()
},
}
_ = common.StartKubeExplorer(explorerCmd.Context(), clusterID, explorerPort)
go func() {
logrus.Infof("autok3s serving kube-explorer on %s", server.Addr)
_ = server.ListenAndServe()
}()
<-wait
_ = server.Shutdown(context.Background())
}

return explorerCmd
Expand Down
4 changes: 1 addition & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ require (
)

require (
github.com/Microsoft/go-winio v0.6.2
github.com/moby/sys/signal v0.7.0
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00
sigs.k8s.io/yaml v1.4.0
Expand All @@ -71,7 +72,6 @@ require (
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
Expand Down Expand Up @@ -190,11 +190,9 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.16.1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/grpc v1.59.0 // indirect
Expand Down
6 changes: 2 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
Expand Down Expand Up @@ -594,8 +594,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down
7 changes: 3 additions & 4 deletions hack/make-rules/autok3s.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,8 @@ function build() {
go_generate
autok3s::log::info "building autok3s(${GIT_VERSION},${GIT_COMMIT},${GIT_TREE_STATE},${BUILD_DATE})..."
# TODO default k3s version in k3d should also get from k3d in
local K3D_VERSION=`go list -m all | grep k3d/v5 | awk '{print $2}'`
local K3S_TAG=`curl --silent --retry 3 "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/'`

local K3D_VERSION=$(go list -m github.com/k3d-io/k3d/v5 | awk '{print $2}')
local K3S_TAG=$(curl --silent --retry 3 'https://update.k3s.io/v1-release/channels/stable' | grep -E -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/')
local version_flags="
-X main.gitVersion=${GIT_VERSION}
-X main.gitCommit=${GIT_COMMIT}
Expand All @@ -81,7 +80,7 @@ function build() {
-X k8s.io/component-base/version.gitCommit=${GIT_COMMIT}
-X k8s.io/component-base/version.gitTreeState=${GIT_TREE_STATE}
-X k8s.io/component-base/version.buildDate=${BUILD_DATE}
-X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.4}
-X github.com/k3d-io/k3d/v5/version.Version=${K3D_VERSION:-v5.4.6}
-X github.com/k3d-io/k3d/v5/version.K3sVersion=${K3S_TAG}"
local flags="
-w -s"
Expand Down
5 changes: 1 addition & 4 deletions pkg/cluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,10 @@ func (p *ProviderBase) InitK3sCluster(cluster *types.Cluster, deployCCM func() [
for plugin := range enabledPlugins {
if plugin == "explorer" {
// start kube-explorer
port, err := common.EnableExplorer(context.Background(), cluster.ContextName)
err := common.EnableExplorer(context.Background(), cluster.ContextName)
if err != nil {
p.Logger.Errorf("[%s] failed to start kube-explorer for cluster %s: %v", p.Provider, cluster.ContextName, err)
}
if port != 0 {
p.Logger.Infof("[%s] kube-explorer for cluster %s will listen on 127.0.0.1:%d...", p.Provider, cluster.Name, port)
}
}
}

Expand Down
84 changes: 39 additions & 45 deletions pkg/common/explorer.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
"os"
"os/exec"
"path/filepath"
"time"

k3dutil "github.com/k3d-io/k3d/v5/cmd/util"
"github.com/sirupsen/logrus"
)

Expand All @@ -16,87 +16,73 @@ const (
)

// EnableExplorer will start kube-explorer with random port for specified K3s cluster
func EnableExplorer(ctx context.Context, config string) (int, error) {
func EnableExplorer(ctx context.Context, config string) error {
if _, ok := ExplorerWatchers[config]; ok {
return 0, fmt.Errorf("kube-explorer for cluster %s has already started", config)
return fmt.Errorf("kube-explorer for cluster %s has already started", config)
}
if err := CheckCommandExist(KubeExplorerCommand); err != nil {
return 0, err
return err
}

// command execution validate
if err := checkExplorerCmd(); err != nil {
return 0, err
return err
}

// save config for kube-explorer
exp, err := DefaultDB.GetExplorer(config)
if err != nil {
return 0, err
return err
}

if exp == nil || !exp.Enabled {
var port int
if exp == nil {
port, err = k3dutil.GetFreePort()
if err != nil {
return 0, err
}
} else {
port = exp.Port
}
exp = &Explorer{
ContextName: config,
Port: port,
Enabled: true,
}
if err = DefaultDB.SaveExplorer(exp); err != nil {
return 0, err
return err
}
}

// start kube-explorer
explorerCtx, cancel := context.WithCancel(ctx)
ExplorerWatchers[config] = cancel
go func(ctx context.Context, config string, port int) {
_ = StartKubeExplorer(ctx, config, port)
}(explorerCtx, config, exp.Port)
return exp.Port, nil

if _, err := StartKubeExplorer(explorerCtx, config); err != nil {
return err
}

return nil
}

// DisableExplorer will stop kube-explorer server for specified K3s cluster
func DisableExplorer(config string) error {
if _, ok := ExplorerWatchers[config]; !ok {
cancelFunc, ok := ExplorerWatchers[config]
if !ok {
return fmt.Errorf("cann't disable unactive kube-explorer for cluster %s", config)
}

// update kube-explorer settings
exp, err := DefaultDB.GetExplorer(config)
if err != nil {
return err
}
if exp != nil && exp.Enabled {
// stop kube-explorer
cancelFunc()
delete(ExplorerWatchers, config)
}
if exp == nil || exp.Enabled {
var port int
if exp == nil {
port, err = k3dutil.GetFreePort()
if err != nil {
return err
}
} else {
port = exp.Port
}
err = DefaultDB.SaveExplorer(&Explorer{
ContextName: config,
Port: port,
Enabled: false,
})
if err != nil {
return err
}
}

// stop kube-explorer
ExplorerWatchers[config]()
delete(ExplorerWatchers, config)
return nil
}

Expand All @@ -110,26 +96,34 @@ func InitExplorer(ctx context.Context) {
for _, exp := range expList {
if exp.Enabled {
logrus.Infof("start kube-explorer for cluster %s", exp.ContextName)
go func(ctx context.Context, name string) {
if _, err = EnableExplorer(ctx, name); err != nil {
logrus.Errorf("failed to start kube-explorer for cluster %s: %v", name, err)
}
}(ctx, exp.ContextName)
if err = EnableExplorer(ctx, exp.ContextName); err != nil {
logrus.Errorf("failed to start kube-explorer for cluster %s: %v", exp.ContextName, err)
}
}
}
}

// StartKubeExplorer start kube-explorer server listen on specified port
func StartKubeExplorer(ctx context.Context, config string, port int) error {
func StartKubeExplorer(ctx context.Context, clusterID string) (chan int, error) {
socketName := GetSocketName(clusterID)
explorer := exec.CommandContext(ctx, KubeExplorerCommand, fmt.Sprintf("--kubeconfig=%s", filepath.Join(CfgPath, KubeCfgFile)),
fmt.Sprintf("--context=%s", config), fmt.Sprintf("--http-listen-port=%d", port), "--https-listen-port=0")
fmt.Sprintf("--context=%s", clusterID), fmt.Sprintf("--bind-address=%s", socketName))
explorer.Stdout = os.Stdout
explorer.Stderr = os.Stderr
explorer.Cancel = func() error {
return explorer.Process.Signal(os.Interrupt)
}
explorer.WaitDelay = 10 * time.Second
if err := explorer.Start(); err != nil {
logrus.Errorf("fail to start kube-explorer for cluster %s: %v", config, err)
logrus.Errorf("fail to start kube-explorer for cluster %s: %v", clusterID, err)
}
logrus.Infof("kube-explorer for %s K3s cluster will listen on 127.0.0.1:%d ...", config, port)
return explorer.Wait()
logrus.Infof("kube-explorer for %s K3s cluster will listen on %s ...", clusterID, socketName)
stopChan := make(chan int)
go func() {
_ = explorer.Wait()
close(stopChan)
}()
return stopChan, nil
}

func CheckCommandExist(cmd string) error {
Expand Down
29 changes: 29 additions & 0 deletions pkg/common/explorer_unix.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
//go:build unix
// +build unix

package common

import (
"context"
"fmt"
"net"
"net/url"
"path/filepath"
)

const (
socketFileName = "socket.sock"
)

func GetSocketName(clusterID string) string {
return fmt.Sprintf("unix://%s", filepath.Join(GetClusterContextPath(clusterID), socketFileName))
}

func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) {
return func(ctx context.Context, _, addr string) (net.Conn, error) {
clusterID, _, _ := net.SplitHostPort(addr)
u, _ := url.Parse(GetSocketName(clusterID))
var d net.Dialer
return d.DialContext(ctx, "unix", u.Path)
}
}
33 changes: 33 additions & 0 deletions pkg/common/explorer_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
//go:build windows
// +build windows

package common

import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"net"
"net/url"

"github.com/Microsoft/go-winio"
)

func GetSocketName(clusterID string) string {
return fmt.Sprintf("namedpipe:/\\.\\pipe\\autok3s-%s", md5hash(clusterID))
}

func GetSocketDialer() func(context.Context, string, string) (net.Conn, error) {
return func(ctx context.Context, _, addr string) (net.Conn, error) {
clusterID, _, _ := net.SplitHostPort(addr)
u, _ := url.Parse(GetSocketName(clusterID))
return winio.DialPipeContext(ctx, u.Path)
}
}

func md5hash(s string) string {
hash := md5.Sum([]byte(s))
hexStr := hex.EncodeToString(hash[:])
return hexStr[:16]
}
Loading

0 comments on commit 66d8b5a

Please sign in to comment.