Merge pull request #366 from xiaods/dev

update code

cmd/encrypt/main.go

@@ -15,14 +15,14 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
-			cmds.NewSecretsEncryptSubcommands(
-				secretsencrypt.Status,
-				secretsencrypt.Enable,
-				secretsencrypt.Disable,
-				secretsencrypt.Prepare,
-				secretsencrypt.Rotate,
-				secretsencrypt.Reencrypt),
+		cmds.NewSecretsEncryptCommands(
+			secretsencrypt.Status,
+			secretsencrypt.Enable,
+			secretsencrypt.Disable,
+			secretsencrypt.Prepare,
+			secretsencrypt.Rotate,
+			secretsencrypt.Reencrypt,
+			secretsencrypt.RotateKeys,
 		),
 	}
 

cmd/etcdsnapshot/main.go

@@ -15,12 +15,11 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewEtcdSnapshotCommand(etcdsnapshot.Save,
-			cmds.NewEtcdSnapshotSubcommands(
-				etcdsnapshot.Delete,
-				etcdsnapshot.List,
-				etcdsnapshot.Prune,
-				etcdsnapshot.Save),
+		cmds.NewEtcdSnapshotCommands(
+			etcdsnapshot.Delete,
+			etcdsnapshot.List,
+			etcdsnapshot.Prune,
+			etcdsnapshot.Save,
 		),
 	}
 

pkg/crd/crds.go

@@ -1,19 +1,19 @@
 package crd
 
 import (
-	"github.com/rancher/wrangler/pkg/crd"
+	"github.com/rancher/wrangler/v3/pkg/crd"
 	v1 "github.com/xiaods/k8e/pkg/apis/k8e.cattle.io/v1"
 )
 
 func List() []crd.CRD {
 	addon := v1.Addon{}
 	etcdSnapshotFile := v1.ETCDSnapshotFile{}
 	return []crd.CRD{
-		crd.NamespacedType("Addon.k8e.cattle.io/v1").
+		crd.NamespacedType("Addon.k3s.cattle.io/v1").
 			WithSchemaFromStruct(addon).
 			WithColumn("Source", ".spec.source").
 			WithColumn("Checksum", ".spec.checksum"),
-		crd.NonNamespacedType("ETCDSnapshotFile.k8e.cattle.io/v1").
+		crd.NonNamespacedType("ETCDSnapshotFile.k3s.cattle.io/v1").
 			WithSchemaFromStruct(etcdSnapshotFile).
 			WithColumn("SnapshotName", ".spec.snapshotName").
 			WithColumn("Node", ".spec.nodeName").

pkg/deploy/controller.go

@@ -16,17 +16,16 @@ import (
 	"time"
 
 	errors2 "github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/apply"
-	"github.com/rancher/wrangler/pkg/kv"
-	"github.com/rancher/wrangler/pkg/merr"
-	"github.com/rancher/wrangler/pkg/objectset"
+	"github.com/rancher/wrangler/v3/pkg/apply"
+	"github.com/rancher/wrangler/v3/pkg/kv"
+	"github.com/rancher/wrangler/v3/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/objectset"
 	"github.com/sirupsen/logrus"
 	"github.com/xiaods/k8e/pkg/agent/util"
 	apisv1 "github.com/xiaods/k8e/pkg/apis/k8e.cattle.io/v1"
 	controllersv1 "github.com/xiaods/k8e/pkg/generated/controllers/k8e.cattle.io/v1"
 	pkgutil "github.com/xiaods/k8e/pkg/util"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -120,6 +119,26 @@ func (w *watcher) listFilesIn(base string, force bool) error {
 		if err != nil {
 			return err
 		}
+		// Descend into symlinked directories, however, only top-level links are followed
+		if info.Mode()&os.ModeSymlink != 0 {
+			linkInfo, err := os.Stat(path)
+			if err != nil {
+				return err
+			}
+			if linkInfo.IsDir() {
+				evalPath, err := filepath.EvalSymlinks(path)
+				if err != nil {
+					return err
+				}
+				filepath.Walk(evalPath, func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						return err
+					}
+					files[path] = info
+					return nil
+				})
+			}
+		}
 		files[path] = info
 		return nil
 	}); err != nil {
@@ -287,7 +306,7 @@ func (w *watcher) delete(path string) error {
 	// ensure that the addon is completely removed before deleting the objectSet,
 	// so return when err == nil, otherwise pods may get stuck terminating
 	w.recorder.Eventf(&addon, corev1.EventTypeNormal, "DeletingManifest", "Deleting manifest at %q", path)
-	if err := w.addons.Delete(addon.Namespace, addon.Name, &metav1.DeleteOptions{}); err == nil || !errors.IsNotFound(err) {
+	if err := w.addons.Delete(addon.Namespace, addon.Name, &metav1.DeleteOptions{}); err == nil || !apierrors.IsNotFound(err) {
 		return err
 	}
 
@@ -303,7 +322,7 @@ func (w *watcher) delete(path string) error {
 // if it cannot be found.
 func (w *watcher) getOrCreateAddon(name string) (apisv1.Addon, error) {
 	addon, err := w.addonCache.Get(metav1.NamespaceSystem, name)
-	if errors.IsNotFound(err) {
+	if apierrors.IsNotFound(err) {
 		addon = apisv1.NewAddon(metav1.NamespaceSystem, name, apisv1.Addon{})
 	} else if err != nil {
 		return apisv1.Addon{}, err

pkg/rootless/mounts.go

@@ -6,6 +6,7 @@ package rootless
 import (
 	"fmt"
 	"os"
+	"os/user"
 	"path/filepath"
 	"strings"
 
@@ -25,11 +26,17 @@ func setupMounts(stateDir string) error {
 		_ = os.RemoveAll(f)
 	}
 
+	runDir, err := resolveRunDir()
+	if err != nil {
+		return err
+	}
+
 	mountMap := [][]string{
 		{"/var/log", filepath.Join(stateDir, "logs")},
 		{"/var/lib/cni", filepath.Join(stateDir, "cni")},
 		{"/var/lib/kubelet", filepath.Join(stateDir, "kubelet")},
-		{"/etc/k8e", filepath.Join(stateDir, "etc", "k8e")},
+		{"/etc/rancher", filepath.Join(stateDir, "etc", "rancher")},
+		{"/run/k3s/containerd", filepath.Join(runDir, "k3s", "containerd")},
 	}
 
 	for _, v := range mountMap {
@@ -91,3 +98,15 @@ func setupMount(target, dir string) error {
 	logrus.Debug("Mounting ", dir, target, " none bind")
 	return unix.Mount(dir, target, "none", unix.MS_BIND, "")
 }
+
+func resolveRunDir() (string, error) {
+	runDir := os.Getenv("XDG_RUNTIME_DIR")
+	if runDir == "" {
+		u, err := user.Lookup(os.Getenv("USER"))
+		if err != nil {
+			return "", err
+		}
+		runDir = filepath.Join("/run/user", u.Uid)
+	}
+	return runDir, nil
+}

pkg/rootlessports/controller.go

@@ -7,7 +7,7 @@ import (
 	"context"
 	"time"
 
-	coreClients "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	coreClients "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/rootless-containers/rootlesskit/pkg/api/client"
 	"github.com/rootless-containers/rootlesskit/pkg/port"
 	"github.com/sirupsen/logrus"
@@ -20,7 +20,7 @@ var (
 	all = "_all_"
 )
 
-func Register(ctx context.Context, serviceController coreClients.ServiceController, enabled bool, httpsPort int) error {
+func Register(ctx context.Context, serviceController coreClients.ServiceController, httpsPort int) error {
 	var (
 		err            error
 		rootlessClient client.Client
@@ -44,7 +44,6 @@ func Register(ctx context.Context, serviceController coreClients.ServiceControll
 	}
 
 	h := &handler{
-		enabled:        enabled,
 		rootlessClient: rootlessClient,
 		serviceClient:  serviceController,
 		serviceCache:   serviceController.Cache(),
@@ -143,11 +142,14 @@ func (h *handler) toBindPorts() (map[int]int, error) {
 					continue
 				}
 
-				if port.Port != 0 {
-					if port.Port <= 1024 {
-						toBindPorts[10000+int(port.Port)] = int(port.Port)
+				for _, toBindPort := range []int32{port.Port, port.NodePort} {
+					if toBindPort == 0 {
+						continue
+					}
+					if toBindPort <= 1024 {
+						toBindPorts[10000+int(toBindPort)] = int(toBindPort)
 					} else {
-						toBindPorts[int(port.Port)] = int(port.Port)
+						toBindPorts[int(toBindPort)] = int(toBindPort)
 					}
 				}
 			}

pkg/server/cert.go

@@ -17,25 +17,26 @@ import (
 
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
-	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/v3/pkg/merr"
 	"github.com/sirupsen/logrus"
 	"github.com/xiaods/k8e/pkg/bootstrap"
 	"github.com/xiaods/k8e/pkg/cluster"
 	"github.com/xiaods/k8e/pkg/daemons/config"
 	"github.com/xiaods/k8e/pkg/daemons/control/deps"
+	"github.com/xiaods/k8e/pkg/util"
 	"github.com/xiaods/k8e/pkg/version"
 	"k8s.io/client-go/util/keyutil"
 )
 
 func caCertReplaceHandler(server *config.Control) http.HandlerFunc {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if req.TLS == nil || req.Method != http.MethodPut {
-			resp.WriteHeader(http.StatusNotFound)
+		if req.Method != http.MethodPut {
+			util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed)
 			return
 		}
 		force, _ := strconv.ParseBool(req.FormValue("force"))
 		if err := caCertReplace(server, req.Body, force); err != nil {
-			genErrorMessage(resp, http.StatusInternalServerError, err, "certificate")
+			util.SendErrorWithID(err, "certificate", resp, req, http.StatusInternalServerError)
 			return
 		}
 		logrus.Infof("certificate: Cluster Certificate Authority data has been updated, %s must be restarted.", version.Program)
@@ -48,7 +49,7 @@ func caCertReplaceHandler(server *config.Control) http.HandlerFunc {
 // the datastore.  If the functions succeeds, servers should be restarted immediately to load the new certs
 // from the bootstrap data.
 func caCertReplace(server *config.Control, buf io.ReadCloser, force bool) error {
-	tmpdir, err := os.MkdirTemp("", "cacerts")
+	tmpdir, err := os.MkdirTemp(server.DataDir, ".rotate-ca-tmp-")
 	if err != nil {
 		return err
 	}
@@ -74,6 +75,10 @@ func caCertReplace(server *config.Control, buf io.ReadCloser, force bool) error
 		return err
 	}
 
+	if err := defaultBootstrap(server, tmpServer); err != nil {
+		return errors.Wrap(err, "failed to set default bootstrap values")
+	}
+
 	if err := validateBootstrap(server, tmpServer); err != nil {
 		if !force {
 			return errors.Wrap(err, "failed to validate new CA certificates and keys")
@@ -84,41 +89,62 @@ func caCertReplace(server *config.Control, buf io.ReadCloser, force bool) error
 	return cluster.Save(context.TODO(), tmpServer, true)
 }
 
-// validateBootstrap checks the new certs and keys to ensure that the cluster would function properly were they to be used.
-// - The new leaf CA certificates must be verifiable using the same root and intermediate certs as the current leaf CA certificates.
-// - The new service account signing key bundle must include the currently active signing key.
-func validateBootstrap(oldServer, newServer *config.Control) error {
+// defaultBootstrap provides default values from the existing bootstrap fields
+// if the value is not tagged for rotation, or the current value is empty.
+func defaultBootstrap(oldServer, newServer *config.Control) error {
 	errs := []error{}
-
 	// Use reflection to iterate over all of the bootstrap fields, checking files at each of the new paths.
 	oldMeta := reflect.ValueOf(&oldServer.Runtime.ControlRuntimeBootstrap).Elem()
 	newMeta := reflect.ValueOf(&newServer.Runtime.ControlRuntimeBootstrap).Elem()
+
+	// use the existing file if the new file does not exist or is empty
 	for _, field := range reflect.VisibleFields(oldMeta.Type()) {
-		oldVal := oldMeta.FieldByName(field.Name)
 		newVal := newMeta.FieldByName(field.Name)
-
 		info, err := os.Stat(newVal.String())
 		if err != nil && !errors.Is(err, fs.ErrNotExist) {
 			errs = append(errs, errors.Wrap(err, field.Name))
 			continue
 		}
 
-		if info == nil || info.Size() == 0 {
+		if field.Tag.Get("rotate") != "true" || info == nil || info.Size() == 0 {
 			if newVal.CanSet() {
-				logrus.Infof("certificate: %s not provided; using current value", field.Name)
+				oldVal := oldMeta.FieldByName(field.Name)
+				logrus.Infof("Using current data for %s: %s", field.Name, oldVal)
 				newVal.Set(oldVal)
 			} else {
 				errs = append(errs, fmt.Errorf("cannot use current data for %s; field is not settable", field.Name))
 			}
 		}
+	}
+	return merr.NewErrors(errs...)
+}
+
+// validateBootstrap checks the new certs and keys to ensure that the cluster would function properly were they to be used.
+// - The new leaf CA certificates must be verifiable using the same root and intermediate certs as the current leaf CA certificates.
+// - The new service account signing key bundle must include the currently active signing key.
+func validateBootstrap(oldServer, newServer *config.Control) error {
+	errs := []error{}
+
+	// Use reflection to iterate over all of the bootstrap fields, checking files at each of the new paths.
+	oldMeta := reflect.ValueOf(&oldServer.Runtime.ControlRuntimeBootstrap).Elem()
+	newMeta := reflect.ValueOf(&newServer.Runtime.ControlRuntimeBootstrap).Elem()
+
+	for _, field := range reflect.VisibleFields(oldMeta.Type()) {
+		// Only handle bootstrap fields tagged for rotation
+		if field.Tag.Get("rotate") != "true" {
+			continue
+		}
+		oldVal := oldMeta.FieldByName(field.Name)
+		newVal := newMeta.FieldByName(field.Name)
 
 		// Check CA chain consistency and cert/key agreement
 		if strings.HasSuffix(field.Name, "CA") {
 			if err := validateCA(oldVal.String(), newVal.String()); err != nil {
 				errs = append(errs, errors.Wrap(err, field.Name))
 			}
 			newKeyVal := newMeta.FieldByName(field.Name + "Key")
-			if err := validateCAKey(newVal.String(), newKeyVal.String()); err != nil {
+			oldKeyVal := oldMeta.FieldByName(field.Name + "Key")
+			if err := validateCAKey(oldVal.String(), oldKeyVal.String(), newVal.String(), newKeyVal.String()); err != nil {
 				errs = append(errs, errors.Wrap(err, field.Name+"Key"))
 			}
 		}
@@ -131,13 +157,15 @@ func validateBootstrap(oldServer, newServer *config.Control) error {
 		}
 	}
 
-	if len(errs) > 0 {
-		return merr.NewErrors(errs...)
-	}
-	return nil
+	return merr.NewErrors(errs...)
 }
 
 func validateCA(oldCAPath, newCAPath string) error {
+	// Skip validation if old values are being reused
+	if oldCAPath == newCAPath {
+		return nil
+	}
+
 	oldCerts, err := certutil.CertsFromFile(oldCAPath)
 	if err != nil {
 		return err
@@ -149,12 +177,12 @@ func validateCA(oldCAPath, newCAPath string) error {
 	}
 
 	if len(newCerts) == 1 {
-		return errors.New("new CA is self-signed")
+		return errors.New("new CA bundle contains only a single certificate but should include root or intermediate CA certificates")
 	}
 
 	roots := x509.NewCertPool()
 	intermediates := x509.NewCertPool()
-	
+
 	// Load all certs from the old bundle
 	for _, cert := range oldCerts {
 		if len(cert.AuthorityKeyId) == 0 || bytes.Equal(cert.AuthorityKeyId, cert.SubjectKeyId) {
@@ -182,7 +210,12 @@ func validateCA(oldCAPath, newCAPath string) error {
 }
 
 // validateCAKey confirms that the private key is valid for the certificate
-func validateCAKey(newCAPath, newCAKeyPath string) error {
+func validateCAKey(oldCAPath, oldCAKeyPath, newCAPath, newCAKeyPath string) error {
+	// Skip validation if old values are being reused
+	if oldCAPath == newCAPath && oldCAKeyPath == newCAKeyPath {
+		return nil
+	}
+
 	_, err := tls.LoadX509KeyPair(newCAPath, newCAKeyPath)
 	if err != nil {
 		err = errors.Wrap(err, "new CA cert and key cannot be loaded as X590KeyPair")