Merge pull request #7454 from maxcao13/reload-ca

Allow VPA admission controller to reload the caBundle certificate and patch the webhook

vertical-pod-autoscaler/deploy/vpa-rbac.yaml

@@ -291,6 +291,7 @@ rules:
       - delete
       - get
       - list
+      - patch
   - apiGroups:
       - "poc.autoscaling.k8s.io"
     resources:

vertical-pod-autoscaler/e2e/v1/admission_controller.go

@@ -853,19 +853,20 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
 		gomega.Expect(err2.Error()).To(gomega.MatchRegexp(`.*admission webhook .*vpa.* denied the request: .*`))
 	})
 
-	ginkgo.It("reloads the webhook certificate", func(ctx ginkgo.SpecContext) {
-		ginkgo.By("Retrieving alternative certificate")
+	ginkgo.It("reloads the webhook leaf and CA certificate", func(ctx ginkgo.SpecContext) {
+		ginkgo.By("Retrieving alternative certificates")
 		c := f.ClientSet
 		e2eCertsSecret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(ctx, "vpa-e2e-certs", metav1.GetOptions{})
 		gomega.Expect(err).To(gomega.Succeed(), "Failed to get vpa-e2e-certs secret")
 		actualCertsSecret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(ctx, "vpa-tls-certs", metav1.GetOptions{})
 		gomega.Expect(err).To(gomega.Succeed(), "Failed to get vpa-tls-certs secret")
 		actualCertsSecret.Data["serverKey.pem"] = e2eCertsSecret.Data["e2eKey.pem"]
 		actualCertsSecret.Data["serverCert.pem"] = e2eCertsSecret.Data["e2eCert.pem"]
+		actualCertsSecret.Data["caCert.pem"] = e2eCertsSecret.Data["e2eCaCert.pem"]
 		_, err = c.CoreV1().Secrets(metav1.NamespaceSystem).Update(ctx, actualCertsSecret, metav1.UpdateOptions{})
 		gomega.Expect(err).To(gomega.Succeed(), "Failed to update vpa-tls-certs secret with e2e rotation certs")
 
-		ginkgo.By("Waiting for certificate reload")
+		ginkgo.By("Waiting for certificate reloads")
 		pods, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(ctx, metav1.ListOptions{})
 		gomega.Expect(err).To(gomega.Succeed())
 
@@ -883,7 +884,7 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
 			logs, err := io.ReadAll(reader)
 			g.Expect(err).To(gomega.Succeed())
 			return string(logs)
-		}).Should(gomega.ContainSubstring("New certificate found, reloading"))
+		}).Should(gomega.And(gomega.ContainSubstring("New certificate found, reloading"), gomega.ContainSubstring("New client CA found, reloading and patching webhook"), gomega.ContainSubstring("Successfully patched webhook with new client CA")))
 
 		ginkgo.By("Setting up invalid VPA object")
 		// there is an invalid "requests" field.

vertical-pod-autoscaler/pkg/admission-controller/certs.go

@@ -17,12 +17,18 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
+	"encoding/base64"
+	"fmt"
 	"os"
-	"path"
 	"sync"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/fsnotify/fsnotify"
+	admissionregistrationv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1"
 	"k8s.io/klog/v2"
 )
 
@@ -42,10 +48,12 @@ func readFile(filePath string) []byte {
 }
 
 type certReloader struct {
-	tlsCertPath string
-	tlsKeyPath  string
-	cert        *tls.Certificate
-	mu          sync.RWMutex
+	tlsCertPath           string
+	tlsKeyPath            string
+	clientCaPath          string
+	cert                  *tls.Certificate
+	mu                    sync.RWMutex
+	mutatingWebhookClient admissionregistrationv1.MutatingWebhookConfigurationInterface
 }
 
 func (cr *certReloader) start(stop <-chan struct{}) error {
@@ -54,22 +62,43 @@ func (cr *certReloader) start(stop <-chan struct{}) error {
 		return err
 	}
 
-	if err = watcher.Add(path.Dir(cr.tlsCertPath)); err != nil {
+	if err = watcher.Add(cr.tlsCertPath); err != nil {
+		return err
+	}
+	if err = watcher.Add(cr.tlsKeyPath); err != nil {
 		return err
 	}
-	if err = watcher.Add(path.Dir(cr.tlsKeyPath)); err != nil {
+	if err = watcher.Add(cr.clientCaPath); err != nil {
 		return err
 	}
+
 	go func() {
 		defer watcher.Close()
 		for {
 			select {
 			case event := <-watcher.Events:
-				if event.Has(fsnotify.Create) || event.Has(fsnotify.Write) {
+				// we need to watch "Remove" events because Kubernetes uses symbolic links to point to ConfigMaps/Secrets volumes
+				if !event.Has(fsnotify.Remove) && !event.Has(fsnotify.Create) && !event.Has(fsnotify.Write) {
+					continue
+				}
+				switch event.Name {
+				case cr.tlsCertPath, cr.tlsKeyPath:
 					klog.V(2).InfoS("New certificate found, reloading")
 					if err := cr.load(); err != nil {
 						klog.ErrorS(err, "Failed to reload certificate")
 					}
+				case cr.clientCaPath:
+					if err := cr.reloadWebhookCA(); err != nil {
+						klog.ErrorS(err, "Failed to reload client CA")
+					}
+				default:
+					continue
+				}
+				// watches get removed along with the symlinks, so we need to add them back
+				if event.Has(fsnotify.Remove) {
+					if err := watcher.Add(event.Name); err != nil {
+						klog.ErrorS(err, "Failed to add watcher for file", "filename", event.Name)
+					}
 				}
 			case err := <-watcher.Errors:
 				klog.Warningf("Error watching certificate files: %s", err)
@@ -92,6 +121,36 @@ func (cr *certReloader) load() error {
 	return nil
 }
 
+func (cr *certReloader) reloadWebhookCA() error {
+	client := cr.mutatingWebhookClient
+	webhook, err := client.Get(context.TODO(), webhookConfigName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if webhook == nil {
+		return fmt.Errorf("webhook not found")
+	}
+	if len(webhook.Webhooks) == 0 {
+		return fmt.Errorf("webhook configuration has no webhooks")
+	}
+	currentBundle := webhook.Webhooks[0].ClientConfig.CABundle[:]
+	base64CurrentBundle := base64.StdEncoding.EncodeToString(currentBundle)
+	newBundle := readFile(cr.clientCaPath)
+	base64NewBundle := base64.StdEncoding.EncodeToString(newBundle)
+	// make sure clientCA actually changed
+	if base64CurrentBundle == base64NewBundle {
+		klog.V(2).InfoS("Client CA did not change, skipping patch")
+		return nil
+	}
+	klog.V(2).InfoS("New client CA found, reloading and patching webhook")
+	patch := []byte(fmt.Sprintf(`{"webhooks":[{"name":"%s","clientConfig":{"caBundle":"%s"}}]}`, webhookName, base64NewBundle))
+	_, err = client.Patch(context.TODO(), webhookConfigName, types.StrategicMergePatchType, patch, metav1.PatchOptions{})
+	if err == nil {
+		klog.V(2).InfoS("Successfully patched webhook with new client CA")
+	}
+	return err
+}
+
 func (cr *certReloader) getCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
 	cr.mu.RLock()
 	defer cr.mu.RUnlock()