Add TLS configuration to Recommender http client (#252)

* Add TLS configuration to Recommender http client

This allows to set client certificate along with server CA allowing
the recommender server to implement mTLS and validate client certificate.
Note that this will work only with a default client implementing an
http.Transport. Any other RoundTripper will need to implement their
own TLS implementation.
The implementation here reloads the certificate and private key for
every request to a given recommender endpoint. This is to support
systems where certificate can be rotated from time to time.

* Implement certificate caching

To prevent reloading the certificate and key from disk at every new
HTTPs connection we cache the certificate in a central cache (up until
it is either expired or too old). The cache is trimmed every 10 minutes
to remove stale entries (not accessed after 10 min).

* Supports setting wpa default TLS config for the recommender API

For environment where certificates can be the same for all WPA resources,
it might be cumbersome to specify the same client certificate in all
the recommender specs.
By using the `--tls-*` option it is possible to set default CA, certificate
and key files (along with a few other TLS options) that will be used
when the recommender specs don't contain such information.

* Retry if certificate & key loaded don't match

For environments where client certificate loaded from disks are renewed
it is very possible that the certificate generator can't write both files
atomically. More care should be used when reading so that if the certificate
and the key don't match, reading is retried until there's either a match or
a failure.

* Declare the recommender tls_config in the CRD

* Refactor tests for a bit more readability

* Review fixes

* Remove LRU mentions as the cache is not an LRU cache
* Cache errors for 1 min to prevent busy looping reading invalid certs

* Review fix: reorder TLS functions

So that types and methods are close together.

* Regenerate all bundle and manifests

Author: masterzen

apis/datadoghq/v1alpha1/watermarkpodautoscaler_types.go

@@ -132,6 +132,38 @@ type WatermarkPodAutoscalerSpec struct {
 	ReadinessDelaySeconds int32 `json:"readinessDelaySeconds,omitempty"`
 }
 
+// TLSConfig specifies the Recommender http client TLS configuration
+// +k8s:openapi-gen=true
+type TLSConfig struct {
+	// CAFile is a path to a CA certificate
+	// +optional
+	CAFile string `json:"caFile,omitempty"`
+
+	// CertFile is a path to a client Cert
+	// +optional
+	CertFile string `json:"certFile,omitempty"`
+
+	// Keyfile is a path to the client certificate key (mandatory if CertFile is set)
+	// +optional
+	KeyFile string `json:"keyFile,omitempty"`
+
+	// ServerName is a settings to activate TLS SNI
+	ServerName string `json:"serverName,omitempty"`
+
+	// InsecureSkipVerify when true disable verifying server certificate
+	InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"`
+
+	// MinVersion, mininum TLS version to accept for the connection. If not set will default to Go default (TLS1.2)
+	// +optional
+	// +kubebuilder:validation:Pattern:=`^TLS1[0-3]$`
+	MinVersion string `json:"minVersion,omitempty"`
+
+	// MaxVersion, maximum TLS version to accept for the connection. If not set will default to Go default (TLS1.2)
+	// +optional
+	// +kubebuilder:validation:Pattern:=`^TLS1[0-3]$`
+	MaxVersion string `json:"maxVersion,omitempty"`
+}
+
 // RecommenderSpec indicates which recommender service to use to calculate the desired replica count
 //
 // See https://github.com/DataDog/agent-payload/pull/348 for details about the API.
@@ -141,6 +173,10 @@ type RecommenderSpec struct {
 	// URL of the recommender service to use
 	URL string `json:"url"`
 
+	// TLS Configuration for the http client allowing to set up a client certificate or server CA certificate
+	// +optional
+	TLSConfig *TLSConfig `json:"tlsConfig,omitempty"`
+
 	// Settings to pass to the recommender service
 	// +optional
 	Settings map[string]string `json:"settings,omitempty"`

apis/datadoghq/v1alpha1/zz_generated.deepcopy.go

@@ -29,10 +29,10 @@ spec:
     - jsonPath: .status.currentMetrics[*].external.currentValue..
       name: value
       type: string
-    - jsonPath: .spec.metrics[*].external.highWatermark..
+    - jsonPath: .spec..highWatermark
       name: high watermark
       type: string
-    - jsonPath: .spec.metrics[*].external.lowWatermark..
+    - jsonPath: .spec..lowWatermark
       name: low watermark
       type: string
     - jsonPath: .metadata.creationTimestamp
@@ -313,6 +313,38 @@ spec:
                     description: TargetType is the type of target the recommender
                       service should use.
                     type: string
+                  tlsConfig:
+                    description: TLS Configuration for the http client allowing to
+                      set up a client certificate or server CA certificate
+                    properties:
+                      caFile:
+                        description: CAFile is a path to a CA certificate
+                        type: string
+                      certFile:
+                        description: CertFile is a path to a client Cert
+                        type: string
+                      insecureSkipVerify:
+                        description: InsecureSkipVerify when true disable verifying
+                          server certificate
+                        type: boolean
+                      keyFile:
+                        description: Keyfile is a path to the client certificate key
+                          (mandatory if CertFile is set)
+                        type: string
+                      maxVersion:
+                        description: MaxVersion, maximum TLS version to accept for
+                          the connection. If not set will default to Go default (TLS1.2)
+                        pattern: ^TLS1[0-3]$
+                        type: string
+                      minVersion:
+                        description: MinVersion, mininum TLS version to accept for
+                          the connection. If not set will default to Go default (TLS1.2)
+                        pattern: ^TLS1[0-3]$
+                        type: string
+                      serverName:
+                        description: ServerName is a settings to activate TLS SNI
+                        type: string
+                    type: object
                   url:
                     description: URL of the recommender service to use
                     type: string

apis/datadoghq/v1alpha1/zz_generated.openapi.go

@@ -313,6 +313,38 @@ spec:
                     description: TargetType is the type of target the recommender
                       service should use.
                     type: string
+                  tlsConfig:
+                    description: TLS Configuration for the http client allowing to
+                      set up a client certificate or server CA certificate
+                    properties:
+                      caFile:
+                        description: CAFile is a path to a CA certificate
+                        type: string
+                      certFile:
+                        description: CertFile is a path to a client Cert
+                        type: string
+                      insecureSkipVerify:
+                        description: InsecureSkipVerify when true disable verifying
+                          server certificate
+                        type: boolean
+                      keyFile:
+                        description: Keyfile is a path to the client certificate key
+                          (mandatory if CertFile is set)
+                        type: string
+                      maxVersion:
+                        description: MaxVersion, maximum TLS version to accept for
+                          the connection. If not set will default to Go default (TLS1.2)
+                        pattern: ^TLS1[0-3]$
+                        type: string
+                      minVersion:
+                        description: MinVersion, mininum TLS version to accept for
+                          the connection. If not set will default to Go default (TLS1.2)
+                        pattern: ^TLS1[0-3]$
+                        type: string
+                      serverName:
+                        description: ServerName is a settings to activate TLS SNI
+                        type: string
+                    type: object
                   url:
                     description: URL of the recommender service to use
                     type: string

bundle/manifests/datadoghq.com_watermarkpodautoscalers.yaml

@@ -303,6 +303,33 @@ spec:
                     targetType:
                       description: TargetType is the type of target the recommender service should use.
                       type: string
+                    tlsConfig:
+                      description: TLS Configuration for the http client allowing to set up a client certificate or server CA certificate
+                      properties:
+                        caFile:
+                          description: CAFile is a path to a CA certificate
+                          type: string
+                        certFile:
+                          description: CertFile is a path to a client Cert
+                          type: string
+                        insecureSkipVerify:
+                          description: InsecureSkipVerify when true disable verifying server certificate
+                          type: boolean
+                        keyFile:
+                          description: Keyfile is a path to the client certificate key (mandatory if CertFile is set)
+                          type: string
+                        maxVersion:
+                          description: MaxVersion, maximum TLS version to accept for the connection. If not set will default to Go default (TLS1.2)
+                          pattern: ^TLS1[0-3]$
+                          type: string
+                        minVersion:
+                          description: MinVersion, mininum TLS version to accept for the connection. If not set will default to Go default (TLS1.2)
+                          pattern: ^TLS1[0-3]$
+                          type: string
+                        serverName:
+                          description: ServerName is a settings to activate TLS SNI
+                          type: string
+                      type: object
                     url:
                       description: URL of the recommender service to use
                       type: string

config/crd/bases/v1/datadoghq.com_watermarkpodautoscalers.yaml

@@ -47,7 +47,22 @@ type RecommenderClient interface {
 }
 
 type RecommenderClientImpl struct {
-	client *http.Client
+	client           *http.Client
+	certificateCache *tlsCertificateCache
+	options          *RecommenderOptions
+}
+
+type RecommenderOption func(*RecommenderOptions)
+
+type RecommenderOptions struct {
+	tlsConfig *v1alpha1.TLSConfig
+}
+
+// WithTLSConfig sets a custom default TLS Config for all Recommender API requests
+func WithTLSConfig(tlsConfig *v1alpha1.TLSConfig) RecommenderOption {
+	return func(option *RecommenderOptions) {
+		option.tlsConfig = tlsConfig
+	}
 }
 
 type ReplicaRecommendationRequest struct {
@@ -72,23 +87,38 @@ type ReplicaRecommendationResponse struct {
 }
 
 // NewRecommenderClient returns a new RecommenderClient with the given http.Client.
-func NewRecommenderClient(client *http.Client) RecommenderClient {
+func NewRecommenderClient(client *http.Client, options ...RecommenderOption) RecommenderClient {
 	if client.Transport == nil {
 		client.Transport = http.DefaultTransport
 	}
+
+	recommenderSettings := &RecommenderOptions{}
+	for _, opt := range options {
+		opt(recommenderSettings)
+	}
+
 	return &RecommenderClientImpl{
-		client: client,
+		client:           client,
+		certificateCache: newTLSCertificateCache(),
+		options:          recommenderSettings,
 	}
 }
 
 // instrumentedClient returns a copy of the client with an instrumented Transport for this recommender.
 //
 // The returned client is a shallow copy of the original client, with the Transport field replaced
 // with an instrumented RoundTripper (which just wraps the original Transport).
-func (r *RecommenderClientImpl) instrumentedClient(recommender string) *http.Client {
+func (r *RecommenderClientImpl) instrumentedClient(recommender string, tlsConfig *v1alpha1.TLSConfig) (*http.Client, error) {
 	client := *r.client
+	if transport, ok := client.Transport.(*http.Transport); ok && tlsConfig != nil {
+		tlsTransport, err := NewCertificateReloadingTransport(tlsConfig, r.certificateCache, transport)
+		if err != nil {
+			return nil, fmt.Errorf("impossible to setup TLS config: %w", err)
+		}
+		client.Transport = tlsTransport
+	}
 	client.Transport = instrumentRoundTripper(recommender, client.Transport)
-	return &client
+	return &client, nil
 }
 
 func instrumentRoundTripper(recommender string, transport http.RoundTripper) http.RoundTripper {
@@ -139,7 +169,10 @@ func (r *RecommenderClientImpl) GetReplicaRecommendation(request *ReplicaRecomme
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	client := r.instrumentedClient(request.Recommender.URL)
+	client, err := r.instrumentedClient(request.Recommender.URL, mergeTLSConfig(r.options.tlsConfig, request.Recommender.TLSConfig))
+	if err != nil {
+		return nil, fmt.Errorf("error creating http client: %w", err)
+	}
 
 	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewReader(payload))
 	httpReq.Header.Set("Content-Type", "application/json")