object: also use system certs for validating RGW cert

When generating the HTTP client used for RGW admin ops, use both system
certs as well as the user-given cert.

As a real world example, admins may use ACME to rotate Letsencrypt certs
every 2 months. For an external CephObjectStore, the cert used by Rook
and RGW may not be rotated at the same time. This can cause the Rook
operator to fail CephObjectStore reconciliation until both certs agree.

When Rook also relies on system certs in the container, Rook's
reconciliation will not have reconciliation failures because
Letsencrypt's well-known and trusted root certificates can be loaded
from the system to validate the RGW's newly-rotated cert.

Signed-off-by: Blaine Gardner <[email protected]>
(cherry picked from commit 7bb72a0)

# Conflicts:
#	Documentation/CRDs/Cluster/external-cluster/provider-export.md
(cherry picked from commit 92267b5)

Documentation/CRDs/Cluster/external-cluster/external-cluster.md

@@ -37,7 +37,7 @@ python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --
 * `--alias-rbd-data-pool-name`: Provides an alias for the  RBD data pool name, necessary if a special character is present in the pool name such as a period or underscore
 * `--rgw-endpoint`: (optional) The RADOS Gateway endpoint in the format `<IP>:<PORT>` or `<FQDN>:<PORT>`.
 * `--rgw-pool-prefix`: (optional) The prefix of the RGW pools. If not specified, the default prefix is `default`
-* `--rgw-tls-cert-path`: (optional) RADOS Gateway endpoint TLS certificate file path
+* `--rgw-tls-cert-path`: (optional) RADOS Gateway endpoint TLS certificate (or intermediate signing certificate) file path
 * `--rgw-skip-tls`: (optional) Ignore TLS certification validation when a self-signed certificate is provided (NOT RECOMMENDED)
 * `--rbd-metadata-ec-pool-name`: (optional) Provides the name of erasure coded RBD metadata pool, used for creating ECRBDStorageClass.
 * `--monitoring-endpoint`: (optional) Ceph Manager prometheus exporter endpoints (comma separated list of IP entries of active and standby mgrs)

deploy/examples/create-external-cluster-resources.py

@@ -377,7 +377,7 @@ def gen_arg_parser(cls, args_to_parse=None):
             "--rgw-tls-cert-path",
             default="",
             required=False,
-            help="RADOS Gateway endpoint TLS certificate",
+            help="RADOS Gateway endpoint TLS certificate (or intermediate signing certificate)",
         )
         output_group.add_argument(
             "--rgw-skip-tls",

pkg/operator/ceph/object/s3-handlers.go

@@ -199,12 +199,18 @@ func (s *S3Agent) DeleteObjectInBucket(bucketname string, key string) (bool, err
 
 func BuildTransportTLS(tlsCert []byte, insecure bool) *http.Transport {
 	//nolint:gosec // is enabled only for testing
-	tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: insecure}
+	tlsConfig := &tls.Config{InsecureSkipVerify: insecure}
+	var caCertPool *x509.CertPool
+	var err error
+	caCertPool, err = x509.SystemCertPool()
+	if err != nil {
+		logger.Warningf("failed to load system cert pool; continuing without loading system certs")
+		caCertPool = x509.NewCertPool() // start with empty cert pool instead
+	}
 	if len(tlsCert) > 0 {
-		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(tlsCert)
-		tlsConfig.RootCAs = caCertPool
 	}
+	tlsConfig.RootCAs = caCertPool
 
 	return &http.Transport{
 		TLSClientConfig: tlsConfig,

pkg/operator/ceph/object/s3-handlers_test.go

@@ -53,7 +53,7 @@ func TestNewS3Agent(t *testing.T) {
 		insecure := true
 		s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure)
 		assert.NoError(t, err)
-		assert.Nil(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.RootCAs)
+		assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.RootCAs) // still includes sys certs
 		assert.True(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify)
 		assert.False(t, *s3Agent.Client.Config.DisableSSL)
 	})