Enable HelmOps deployments with strict TLS mode

e2e/assets/multi-cluster/helmop.yaml

@@ -0,0 +1,16 @@
+apiVersion: fleet.cattle.io/v1alpha1
+kind: HelmOp
+metadata:
+  name: {{.Name}}
+  namespace: {{.Namespace}}
+spec:
+  helm:
+    releaseName: testhelm
+    repo: {{.Repo}}
+    chart: {{.Chart}}
+    version: '{{.Version}}'
+  pollingInterval: {{.PollingInterval}}
+  helmSecretName: {{.HelmSecretName}}
+  insecureSkipTLSVerify: {{.InsecureSkipTLSVerify}}
+  targets:
+    - clusterSelector: {}

e2e/multi-cluster/installation/agent_test.go

@@ -1,10 +1,15 @@
 package installation_test
 
 import (
+	"encoding/json"
 	"fmt"
+	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	"github.com/rancher/fleet/e2e/testenv"
 )
 
 var (
@@ -13,8 +18,6 @@ var (
 
 var _ = Describe("Fleet installation with TLS agent modes", func() {
 	BeforeEach(func() {
-		kd = env.Kubectl.Context(env.Downstream)
-
 		_, err := kd.Delete(
 			"pod",
 			"-n",
@@ -63,17 +66,17 @@ var _ = Describe("Fleet installation with TLS agent modes", func() {
 					g.Expect(logs).To(MatchRegexp("Failed to register agent.*could not find the requested resource"))
 
 					return nil
-				}).ShouldNot(HaveOccurred())
+				}).Should(Succeed())
 			})
 		})
 	})
 
 	Context("with strict agent TLS mode", func() {
-		When("fetching fleet-agent logs", func() {
-			BeforeEach(func() {
-				agentMode = "strict"
-			})
+		BeforeEach(func() {
+			agentMode = "strict"
+		})
 
+		When("fetching fleet-agent logs", func() {
 			It("cannot reach the server because the cert is signed by an unknown authority", func() {
 				Eventually(func(g Gomega) error {
 					logs, err := kd.Namespace("cattle-fleet-system").Logs(
@@ -93,8 +96,88 @@ var _ = Describe("Fleet installation with TLS agent modes", func() {
 					}
 
 					return nil
-				}).ShouldNot(HaveOccurred())
+				}).Should(Succeed())
+			})
+		})
+	})
+})
+
+var _ = Describe("HelmOps installation with strict TLS mode", func() {
+	BeforeEach(func() {
+		_, err := kd.Delete("pod", "-n", "cattle-fleet-system", "-l", "app=fleet-agent")
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	JustBeforeEach(func() {
+		strictCfg := strings.ReplaceAll(config, "system-store", "strict")
+
+		out, err := ku.Patch(
+			"configmap",
+			"fleet-controller",
+			"-n",
+			"cattle-fleet-system",
+			"--type=merge",
+			"-p",
+			fmt.Sprintf(`{"data":{"config":"%s"}}`, strictCfg),
+		)
+		Expect(err).ToNot(HaveOccurred(), string(out))
+
+		// Check that the config change has been applied downstream
+		type configWithTLSMode struct {
+			AgentTLSMode string `json:"agentTLSMode"`
+		}
+		Eventually(func(g Gomega) {
+			data, err := kd.Namespace("cattle-fleet-system").Get("configmap", "fleet-agent", "-o", "jsonpath={.data.config}")
+			g.Expect(err).ToNot(HaveOccurred(), data)
+
+			var c configWithTLSMode
+
+			err = json.Unmarshal([]byte(data), &c)
+			g.Expect(err).ToNot(HaveOccurred())
+
+			g.Expect(c.AgentTLSMode).To(Equal("strict"))
+		}).Should(Succeed())
+	})
+
+	When("installing a HelmOp", func() {
+		name := "basic"
+		ns := "fleet-default"
+
+		JustBeforeEach(func() {
+			ku = ku.Namespace(ns)
+			err := testenv.ApplyTemplate(ku, "../../assets/multi-cluster/helmop.yaml", struct {
+				Name                  string
+				Namespace             string
+				Repo                  string
+				Chart                 string
+				Version               string
+				PollingInterval       time.Duration
+				HelmSecretName        string
+				InsecureSkipTLSVerify bool
+			}{
+				name,
+				ns,
+				"",
+				"https://github.com/rancher/fleet/raw/refs/heads/main/integrationtests/cli/assets/helmrepository/config-chart-0.1.0.tgz",
+				"0.1.0",
+				0,
+				"",
+				false,
 			})
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("deploys the chart", func() {
+			Eventually(func(g Gomega) {
+				outPods, err := kd.Get("configmaps")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(outPods).To(ContainSubstring("test-simple-chart-config"))
+			}).Should(Succeed())
+		})
+
+		AfterEach(func() {
+			_, err := ku.Delete("helmop", name)
+			Expect(err).ToNot(HaveOccurred())
 		})
 	})
 })

internal/cmd/agent/operator.go

@@ -345,9 +345,5 @@ func getAgentConfig(ctx context.Context, namespace string, cfg *rest.Config) (ag
 		return nil, fmt.Errorf("failed to parse config from ConfigMap: %w", err)
 	}
 
-	if agentConfig.AgentTLSMode == config.AgentTLSModeStrict {
-		config.BypassSystemCAStore()
-	}
-
 	return agentConfig, nil
 }

internal/cmd/agent/register/register.go

@@ -140,7 +140,8 @@ func runRegistration(ctx context.Context, k8s coreInterface, namespace string) (
 		return nil, fmt.Errorf("failed to look up client config %s/%s: %w", secret.Namespace, config.AgentConfigName, err)
 	}
 
-	clientConfig := createClientConfigFromSecret(secret, cfg.AgentTLSMode == config.AgentTLSModeSystemStore)
+	clientConfig, undoBypass := createClientConfigFromSecret(secret, cfg.AgentTLSMode == config.AgentTLSModeSystemStore)
+	defer undoBypass()
 
 	ns, _, err := clientConfig.Namespace()
 	if err != nil {
@@ -287,20 +288,22 @@ func values(data map[string][]byte) map[string][]byte {
 
 // createClientConfigFromSecret reads the fleet-agent-bootstrap secret and
 // creates a clientConfig to access the upstream cluster
-func createClientConfigFromSecret(secret *corev1.Secret, trustSystemStoreCAs bool) clientcmd.ClientConfig {
+func createClientConfigFromSecret(secret *corev1.Secret, trustSystemStoreCAs bool) (clientcmd.ClientConfig, func()) {
 	data := values(secret.Data)
 	apiServerURL := string(data[config.APIServerURLKey])
 	apiServerCA := data[config.APIServerCAKey]
 	namespace := string(data[ClusterNamespace])
 	token := string(data[Token])
 
+	undoBypass := func() {}
+
 	if trustSystemStoreCAs { // Save a request to the API server URL if system CAs are not to be trusted.
 		// NOTE(manno): client-go will use the system trust store even if a CA is configured. So, why do this?
 		if _, err := http.Get(apiServerURL); err == nil {
 			apiServerCA = nil
 		}
 	} else {
-		config.BypassSystemCAStore()
+		undoBypass = config.BypassSystemCAStore()
 	}
 
 	cfg := clientcmdapi.Config{
@@ -325,7 +328,7 @@ func createClientConfigFromSecret(secret *corev1.Secret, trustSystemStoreCAs boo
 		CurrentContext: "default",
 	}
 
-	return clientcmd.NewDefaultClientConfig(cfg, &clientcmd.ConfigOverrides{})
+	return clientcmd.NewDefaultClientConfig(cfg, &clientcmd.ConfigOverrides{}), undoBypass
 }
 
 func testClientConfig(cfg []byte) error {

internal/config/overrides.go

@@ -6,20 +6,37 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	envVarSSLCertFile = "SSL_CERT_FILE"
+	envVarSSLCertDir  = "SSL_CERT_DIR"
+)
+
 // BypassSystemCAStore is used to bypass the OS trust store in agents through env vars, see
 // https://pkg.go.dev/crypto/x509#SystemCertPool for more info.
 // We set values to paths belonging to the root filesystem, which is read-only, to prevent tampering.
 // Eventually, this should not be necessary, if/when we find a way to set client-go's API Config to achieve similar
 // effects.
 // Note: this will not work on Windows nor Mac OS. Agents are expected to run on Linux nodes.
-func BypassSystemCAStore() {
-	err := os.Setenv("SSL_CERT_FILE", "/dev/null")
-	if err != nil {
-		logrus.Errorf("failed to set env var SSL_CERT_FILE: %s", err.Error())
+// Returns a function allowing the bypass to be undone.
+func BypassSystemCAStore() func() {
+	certFileBkp := os.Getenv(envVarSSLCertFile)
+	certDirBkp := os.Getenv(envVarSSLCertDir)
+
+	if err := os.Setenv(envVarSSLCertFile, "/dev/null"); err != nil {
+		logrus.Errorf("failed to set env var %s: %s", envVarSSLCertFile, err.Error())
 	}
 
-	err = os.Setenv("SSL_CERT_DIR", "/dev/null")
-	if err != nil {
-		logrus.Errorf("failed to set env var SSL_CERT_DIR: %s", err.Error())
+	if err := os.Setenv(envVarSSLCertDir, "/dev/null"); err != nil {
+		logrus.Errorf("failed to set env var %s: %s", envVarSSLCertDir, err.Error())
+	}
+
+	return func() {
+		if err := os.Setenv(envVarSSLCertFile, certFileBkp); err != nil {
+			logrus.Errorf("failed to restore env var %s: %s", envVarSSLCertFile, err.Error())
+		}
+
+		if err := os.Setenv(envVarSSLCertDir, certDirBkp); err != nil {
+			logrus.Errorf("failed to restore env var %s: %s", envVarSSLCertDir, err.Error())
+		}
 	}
 }