Skip to content

Commit 8227e6d

Browse files
weyfonkweyfonk
committed
Enable HelmOps deployments with strict TLS mode
When the agent TLS mode is set to `strict`, the Fleet agent bypasses the operating system's CA store only for the duration of the agent registration process. Once registration is successful, the store can be used again, which enables Helm charts to be pulled from the agent.
1 parent de37844 commit 8227e6d

File tree

6 files changed

+152
-21
lines changed

6 files changed

+152
-21
lines changed

‎e2e/assets/multi-cluster/helmop.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: fleet.cattle.io/v1alpha1
2+
kind: HelmOp
3+
metadata:
4+
name: {{.Name}}
5+
namespace: {{.Namespace}}
6+
spec:
7+
helm:
8+
releaseName: testhelm
9+
repo: {{.Repo}}
10+
chart: {{.Chart}}
11+
version: '{{.Version}}'
12+
pollingInterval: {{.PollingInterval}}
13+
helmSecretName: {{.HelmSecretName}}
14+
insecureSkipTLSVerify: {{.InsecureSkipTLSVerify}}
15+
targets:
16+
- clusterSelector: {}

‎e2e/multi-cluster/installation/agent_test.go‎

Lines changed: 99 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
package installation_test
22

33
import (
4+
"encoding/json"
45
"fmt"
6+
"strings"
7+
"time"
58

69
. "github.com/onsi/ginkgo/v2"
710
. "github.com/onsi/gomega"
11+
12+
"github.com/rancher/fleet/e2e/testenv"
813
)
914

1015
var (
@@ -69,11 +74,11 @@ var _ = Describe("Fleet installation with TLS agent modes", func() {
6974
})
7075

7176
Context("with strict agent TLS mode", func() {
72-
When("fetching fleet-agent logs", func() {
73-
BeforeEach(func() {
74-
agentMode = "strict"
75-
})
77+
BeforeEach(func() {
78+
agentMode = "strict"
79+
})
7680

81+
When("fetching fleet-agent logs", func() {
7782
It("cannot reach the server because the cert is signed by an unknown authority", func() {
7883
Eventually(func(g Gomega) error {
7984
logs, err := kd.Namespace("cattle-fleet-system").Logs(
@@ -98,3 +103,93 @@ var _ = Describe("Fleet installation with TLS agent modes", func() {
98103
})
99104
})
100105
})
106+
107+
var _ = Describe("HelmOps installation with strict TLS mode", func() {
108+
BeforeEach(func() {
109+
kd = env.Kubectl.Context(env.Downstream)
110+
111+
_, err := kd.Delete(
112+
"pod",
113+
"-n",
114+
"cattle-fleet-system",
115+
"-l",
116+
"app=fleet-agent",
117+
)
118+
Expect(err).NotTo(HaveOccurred())
119+
})
120+
121+
JustBeforeEach(func() {
122+
restoreConfig() // prevent interference with other test cases in the suite.
123+
out, err := ku.Patch(
124+
"configmap",
125+
"fleet-controller",
126+
"-n",
127+
"cattle-fleet-system",
128+
"--type=merge",
129+
"-p",
130+
fmt.Sprintf(`{"data":{"config":"{\"agentTLSMode\": \"%s\"}"}}`, agentMode),
131+
)
132+
Expect(err).ToNot(HaveOccurred(), string(out))
133+
134+
// Check that the config change has been applied downstream
135+
type configWithTLSMode struct {
136+
AgentTLSMode string `json:"agentTLSMode"`
137+
}
138+
Eventually(func(g Gomega) {
139+
data, err := kd.Namespace("cattle-fleet-system").Get(
140+
"configmap",
141+
"fleet-agent",
142+
"-o",
143+
"jsonpath={.data.config}",
144+
)
145+
Expect(err).ToNot(HaveOccurred(), data)
146+
147+
var c configWithTLSMode
148+
149+
err = json.Unmarshal([]byte(data), &c)
150+
Expect(err).ToNot(HaveOccurred())
151+
152+
Expect(c.AgentTLSMode).To(Equal("strict"))
153+
}).Should(Succeed())
154+
})
155+
156+
When("installing a HelmOp", func() {
157+
name := "basic"
158+
ns := "fleet-default"
159+
160+
JustBeforeEach(func() {
161+
ku = ku.Namespace(ns)
162+
err := testenv.ApplyTemplate(ku, "../../assets/multi-cluster/helmop.yaml", struct {
163+
Name string
164+
Namespace string
165+
Repo string
166+
Chart string
167+
Version string
168+
PollingInterval time.Duration
169+
HelmSecretName string
170+
InsecureSkipTLSVerify bool
171+
}{
172+
name,
173+
ns,
174+
"",
175+
"https://github.com/rancher/fleet/raw/refs/heads/main/integrationtests/cli/assets/helmrepository/config-chart-0.1.0.tgz",
176+
"0.1.0",
177+
0,
178+
"",
179+
false,
180+
})
181+
Expect(err).ToNot(HaveOccurred())
182+
})
183+
184+
It("deploys the chart", func() {
185+
Eventually(func() bool {
186+
outPods, _ := kd.Get("configmaps")
187+
return strings.Contains(outPods, "test-simple-chart-config")
188+
}).Should(BeTrue())
189+
})
190+
191+
AfterEach(func() {
192+
ku.Delete("helmop", name)
193+
})
194+
})
195+
})

‎e2e/multi-cluster/installation/suite_test.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,11 @@ var _ = BeforeSuite(func() {
4949
})
5050

5151
var _ = AfterSuite(func() {
52-
// Restore initial state of config map
52+
restoreConfig()
53+
})
54+
55+
// restoreConfig restores the initial state of the `fleet-controller` config map.
56+
func restoreConfig() {
5357
out, err := ku.Patch(
5458
"configmap",
5559
"fleet-controller",
@@ -60,4 +64,4 @@ var _ = AfterSuite(func() {
6064
fmt.Sprintf(`{"data":{"config":"%s"}}`, config),
6165
)
6266
Expect(err).ToNot(HaveOccurred(), string(out))
63-
})
67+
}

‎internal/cmd/agent/operator.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -345,9 +345,5 @@ func getAgentConfig(ctx context.Context, namespace string, cfg *rest.Config) (ag
345345
return nil, fmt.Errorf("failed to parse config from ConfigMap: %w", err)
346346
}
347347

348-
if agentConfig.AgentTLSMode == config.AgentTLSModeStrict {
349-
config.BypassSystemCAStore()
350-
}
351-
352348
return agentConfig, nil
353349
}

‎internal/cmd/agent/register/register.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,8 @@ func runRegistration(ctx context.Context, k8s coreInterface, namespace string) (
140140
return nil, fmt.Errorf("failed to look up client config %s/%s: %w", secret.Namespace, config.AgentConfigName, err)
141141
}
142142

143-
clientConfig := createClientConfigFromSecret(secret, cfg.AgentTLSMode == config.AgentTLSModeSystemStore)
143+
clientConfig, undoBypass := createClientConfigFromSecret(secret, cfg.AgentTLSMode == config.AgentTLSModeSystemStore)
144+
defer undoBypass()
144145

145146
ns, _, err := clientConfig.Namespace()
146147
if err != nil {
@@ -287,20 +288,22 @@ func values(data map[string][]byte) map[string][]byte {
287288

288289
// createClientConfigFromSecret reads the fleet-agent-bootstrap secret and
289290
// creates a clientConfig to access the upstream cluster
290-
func createClientConfigFromSecret(secret *corev1.Secret, trustSystemStoreCAs bool) clientcmd.ClientConfig {
291+
func createClientConfigFromSecret(secret *corev1.Secret, trustSystemStoreCAs bool) (clientcmd.ClientConfig, func()) {
291292
data := values(secret.Data)
292293
apiServerURL := string(data[config.APIServerURLKey])
293294
apiServerCA := data[config.APIServerCAKey]
294295
namespace := string(data[ClusterNamespace])
295296
token := string(data[Token])
296297

298+
undoBypass := func() {}
299+
297300
if trustSystemStoreCAs { // Save a request to the API server URL if system CAs are not to be trusted.
298301
// NOTE(manno): client-go will use the system trust store even if a CA is configured. So, why do this?
299302
if _, err := http.Get(apiServerURL); err == nil {
300303
apiServerCA = nil
301304
}
302305
} else {
303-
config.BypassSystemCAStore()
306+
undoBypass = config.BypassSystemCAStore()
304307
}
305308

306309
cfg := clientcmdapi.Config{
@@ -325,7 +328,7 @@ func createClientConfigFromSecret(secret *corev1.Secret, trustSystemStoreCAs boo
325328
CurrentContext: "default",
326329
}
327330

328-
return clientcmd.NewDefaultClientConfig(cfg, &clientcmd.ConfigOverrides{})
331+
return clientcmd.NewDefaultClientConfig(cfg, &clientcmd.ConfigOverrides{}), undoBypass
329332
}
330333

331334
func testClientConfig(cfg []byte) error {

‎internal/config/overrides.go‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,37 @@ import (
66
"github.com/sirupsen/logrus"
77
)
88

9+
const (
10+
envVarSSLCertFile = "SSL_CERT_FILE"
11+
envVarSSLCertDir = "SSL_CERT_DIR"
12+
)
13+
914
// BypassSystemCAStore is used to bypass the OS trust store in agents through env vars, see
1015
// https://pkg.go.dev/crypto/x509#SystemCertPool for more info.
1116
// We set values to paths belonging to the root filesystem, which is read-only, to prevent tampering.
1217
// Eventually, this should not be necessary, if/when we find a way to set client-go's API Config to achieve similar
1318
// effects.
1419
// Note: this will not work on Windows nor Mac OS. Agents are expected to run on Linux nodes.
15-
func BypassSystemCAStore() {
16-
err := os.Setenv("SSL_CERT_FILE", "/dev/null")
17-
if err != nil {
18-
logrus.Errorf("failed to set env var SSL_CERT_FILE: %s", err.Error())
20+
// Returns a function allowing the bypass to be undone.
21+
func BypassSystemCAStore() func() {
22+
certFileBkp := os.Getenv(envVarSSLCertFile)
23+
certDirBkp := os.Getenv(envVarSSLCertDir)
24+
25+
if err := os.Setenv(envVarSSLCertFile, "/dev/null"); err != nil {
26+
logrus.Errorf("failed to set env var %s: %s", envVarSSLCertFile, err.Error())
1927
}
2028

21-
err = os.Setenv("SSL_CERT_DIR", "/dev/null")
22-
if err != nil {
23-
logrus.Errorf("failed to set env var SSL_CERT_DIR: %s", err.Error())
29+
if err := os.Setenv(envVarSSLCertDir, "/dev/null"); err != nil {
30+
logrus.Errorf("failed to set env var %s: %s", envVarSSLCertDir, err.Error())
31+
}
32+
33+
return func() {
34+
if err := os.Setenv(envVarSSLCertFile, certFileBkp); err != nil {
35+
logrus.Errorf("failed to restore env var %s: %s", envVarSSLCertFile, err.Error())
36+
}
37+
38+
if err := os.Setenv(envVarSSLCertDir, certDirBkp); err != nil {
39+
logrus.Errorf("failed to restore env var %s: %s", envVarSSLCertDir, err.Error())
40+
}
2441
}
2542
}

0 commit comments

Comments
 (0)