feat: add support for multiple SCP composition

Author: pkoutsovasilis

config/crds/v1/all-crds.yaml

@@ -10932,6 +10932,13 @@ spec:
                   - secretName
                   type: object
                 type: array
+              weight:
+                default: 0
+                description: |-
+                  Weight determines the priority of this policy when multiple policies target the same resource.
+                  Lower weight values take precedence. Defaults to 0.
+                format: int32
+                type: integer
             type: object
           status:
             properties:

config/crds/v1/resources/stackconfigpolicy.k8s.elastic.co_stackconfigpolicies.yaml

@@ -288,6 +288,13 @@ spec:
                   - secretName
                   type: object
                 type: array
+              weight:
+                default: 0
+                description: |-
+                  Weight determines the priority of this policy when multiple policies target the same resource.
+                  Lower weight values take precedence. Defaults to 0.
+                format: int32
+                type: integer
             type: object
           status:
             properties:

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -11002,6 +11002,13 @@ spec:
                   - secretName
                   type: object
                 type: array
+              weight:
+                default: 0
+                description: |-
+                  Weight determines the priority of this policy when multiple policies target the same resource.
+                  Lower weight values take precedence. Defaults to 0.
+                format: int32
+                type: integer
             type: object
           status:
             properties:

docs/reference/api-reference/main.md

@@ -2068,6 +2068,7 @@ StackConfigPolicy represents a StackConfigPolicy resource in a Kubernetes cluste
 | Field | Description |
 | --- | --- |
 | *`resourceSelector`* __[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#labelselector-v1-meta)__ |  |
+| *`weight`* __integer__ | Weight determines the priority of this policy when multiple policies target the same resource.<br>Lower weight values take precedence. Defaults to 0. |
 | *`secureSettings`* __[SecretSource](#secretsource) array__ | Deprecated: SecureSettings only applies to Elasticsearch and is deprecated. It must be set per application instead. |
 | *`elasticsearch`* __[ElasticsearchConfigPolicySpec](#elasticsearchconfigpolicyspec)__ |  |
 | *`kibana`* __[KibanaConfigPolicySpec](#kibanaconfigpolicyspec)__ |  |

pkg/apis/stackconfigpolicy/v1alpha1/stackconfigpolicy_types.go

@@ -55,6 +55,10 @@ type StackConfigPolicyList struct {
 
 type StackConfigPolicySpec struct {
 	ResourceSelector metav1.LabelSelector `json:"resourceSelector,omitempty"`
+	// Weight determines the priority of this policy when multiple policies target the same resource.
+	// Lower weight values take precedence. Defaults to 0.
+	// +kubebuilder:default=0
+	Weight int32 `json:"weight,omitempty"`
 	// Deprecated: SecureSettings only applies to Elasticsearch and is deprecated. It must be set per application instead.
 	SecureSettings []commonv1.SecretSource       `json:"secureSettings,omitempty"`
 	Elasticsearch  ElasticsearchConfigPolicySpec `json:"elasticsearch,omitempty"`

pkg/controller/common/annotation/constants.go

@@ -25,4 +25,6 @@ const (
 
 	ElasticsearchConfigAndSecretMountsHashAnnotation = "policy.k8s.elastic.co/elasticsearch-config-mounts-hash" //nolint:gosec
 	SourceSecretAnnotationName                       = "policy.k8s.elastic.co/source-secret-name"               //nolint:gosec
+
+	SoftOwnerRefsAnnotation = "eck.k8s.elastic.co/owner-refs"
 )

pkg/controller/common/reconciler/secret.go

@@ -22,7 +22,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/v3/pkg/utils/maps"
 )
 
-// labels set on secrets which cannot rely on owner references due to https://github.com/kubernetes/kubernetes/issues/65200,
+// labels and annotations set on secrets which cannot rely on owner references due to https://github.com/kubernetes/kubernetes/issues/65200,
 // but should still be garbage-collected (best-effort) by the operator upon owner deletion.
 const (
 	SoftOwnerNamespaceLabel = "eck.k8s.elastic.co/owner-namespace"

pkg/controller/elasticsearch/filesettings/reconciler.go

@@ -29,7 +29,7 @@ var (
 
 	// managedAnnotations are the annotations managed by the operator for the stack config policy related secrets, which means that the operator
 	// will always take precedence to update or remove these annotations.
-	managedAnnotations = []string{commonannotation.SecureSettingsSecretsAnnotationName, commonannotation.SettingsHashAnnotationName, commonannotation.ElasticsearchConfigAndSecretMountsHashAnnotation, commonannotation.KibanaConfigHashAnnotation}
+	managedAnnotations = []string{commonannotation.SecureSettingsSecretsAnnotationName, commonannotation.SettingsHashAnnotationName, commonannotation.ElasticsearchConfigAndSecretMountsHashAnnotation, commonannotation.KibanaConfigHashAnnotation, commonannotation.SoftOwnerRefsAnnotation}
 )
 
 // ReconcileEmptyFileSettingsSecret reconciles an empty File settings Secret for the given Elasticsearch only when there is no Secret.

pkg/controller/elasticsearch/filesettings/secret.go

@@ -85,9 +85,6 @@ func newSettingsSecret(version int64, es types.NamespacedName, currentSecret *co
 	}
 
 	if policy != nil {
-		// set this policy as soft owner of this Secret
-		SetSoftOwner(settingsSecret, *policy)
-
 		// add the Secure Settings Secret sources to the Settings Secret
 		if err := setSecureSettings(settingsSecret, *policy); err != nil {
 			return corev1.Secret{}, 0, err