Remove controlNamespace from resourceOwner and various improvements (#25)

* move GetControlNamespace to separate interface
* add Created desired state
* check restmapper while pruning
* ignore notfound error while pruning
* fix predicates
* extend desiredstate to allow more fine grained reconcile
- BeforeUpdate signature changed and has the actual desired object
- BeforeCreate and BeforeDelete added
- create, update, delete options can be set within desired state
- annotation and label merge removed
- ownership annotation removal also removed
* gracefully check cross-namespace ownership
* fix silent cross-namespace ownership condition
* add create/update/delete pre condition funcs

Co-authored-by: Zsolt Varga <[email protected]>
Co-authored-by: Toader Sebastian <[email protected]>
Co-authored-by: Dudás Ádám <[email protected]>

Author: 4 people

docs/types/Readme.md

@@ -5,7 +5,7 @@ For more information please click on the name
 
 | Name | Description |
 |---|---|
-| **[MetaBase](base_types.md)** |  |
+| **[ObjectKey](base_types.md)** |  |
 | **[Secret](secret_types.md)** | Secret referencing abstraction |
 | **[KubernetesVolume](volume_types.md)** | Kubernetes volume abstraction |
 </center>

docs/types/base_types.md

@@ -1,3 +1,8 @@
+### ObjectKey
+| Variable Name | Type | Required | Default | Description |
+|---|---|---|---|---|
+| name | string | No | - |  |
+| namespace | string | No | - |  |
 ### MetaBase
 | Variable Name | Type | Required | Default | Description |
 |---|---|---|---|---|

pkg/reconciler/component.go

@@ -15,11 +15,13 @@
 package reconciler
 
 import (
+	"emperror.dev/errors"
 	"github.com/go-logr/logr"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
@@ -28,6 +30,10 @@ type ComponentReconciler interface {
 	RegisterWatches(*builder.Builder)
 }
 
+type Watches interface {
+	SetupAdditionalWatches(c controller.Controller) error
+}
+
 // Dispatcher orchestrates reconciliation of multiple ComponentReconciler objects
 // focusing on handing off reconciled object to all of its components and calculating an aggregated result to return.
 // It requires a ResourceGetter callback and optionally can leverage a ResourceFilter and a CompletionHandler
@@ -44,20 +50,20 @@ type Dispatcher struct {
 func (r *Dispatcher) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 	object, err := r.ResourceGetter(req)
 	if err != nil {
-		return reconcile.Result{}, err
+		return reconcile.Result{}, errors.WithStack(err)
 	}
 	if r.ResourceFilter != nil {
 		shouldReconcile, err := r.ResourceFilter(object)
 		if err != nil || !shouldReconcile {
-			return reconcile.Result{}, err
+			return reconcile.Result{}, errors.WithStack(err)
 		}
 	}
 	result, err := r.Handle(object)
 	if r.CompletionHandler != nil {
-		return r.CompletionHandler(object, result, err)
+		return r.CompletionHandler(object, result, errors.WithStack(err))
 	}
 	if err != nil {
-		return result, err
+		return result, errors.WithStack(err)
 	}
 	return result, nil
 }
@@ -68,7 +74,12 @@ func (r *Dispatcher) Handle(object runtime.Object) (ctrl.Result, error) {
 	combinedResult := &CombinedResult{}
 	for _, cr := range r.ComponentReconcilers {
 		result, err := cr.Reconcile(object)
-		combinedResult.Combine(result, err)
+		combinedResult.Combine(result, errors.WithStack(err))
+		if cr, ok := cr.(interface{ IsOptional() bool }); ok {
+			if err != nil && !cr.IsOptional() {
+				break
+			}
+		}
 	}
 	return combinedResult.Result, combinedResult.Err
 }
@@ -80,3 +91,17 @@ func (r *Dispatcher) RegisterWatches(b *builder.Builder) *builder.Builder {
 	}
 	return b
 }
+
+// SetupAdditionalWatches dispatches the controller for watch registration to all its components
+func (r *Dispatcher) SetupAdditionalWatches(c controller.Controller) error {
+	for _, cr := range r.ComponentReconcilers {
+		if cr, ok := cr.(Watches); ok {
+			err := cr.SetupAdditionalWatches(c)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+		}
+	}
+
+	return nil
+}

pkg/reconciler/handlers.go

@@ -0,0 +1,42 @@
+// Copyright © 2020 Banzai Cloud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reconciler
+
+import (
+	"strings"
+
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	ottypes "github.com/banzaicloud/operator-tools/pkg/types"
+)
+
+func EnqueueByOwnerAnnotationMapper() handler.Mapper {
+	return handler.ToRequestsFunc(func(a handler.MapObject) []reconcile.Request {
+		pieces := strings.SplitN(a.Meta.GetAnnotations()[ottypes.BanzaiCloudRelatedTo], string(types.Separator), 2)
+		if len(pieces) != 2 {
+			return []reconcile.Request{}
+		}
+
+		return []reconcile.Request{
+			{NamespacedName: client.ObjectKey{
+				Name:      pieces[1],
+				Namespace: pieces[0],
+			}},
+		}
+	})
+}

pkg/reconciler/native.go

@@ -21,7 +21,9 @@ import (
 	"emperror.dev/errors"
 	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -32,21 +34,28 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-)
 
-const BanzaiCloudManagedComponent = "banzaicloud.io/managed-component"
+	"github.com/banzaicloud/operator-tools/pkg/types"
+	"github.com/banzaicloud/operator-tools/pkg/utils"
+)
 
 type ResourceOwner interface {
 	// to be aware of metadata
 	metav1.Object
 	// to be aware of the owner's type
 	runtime.Object
+}
+
+type ResourceOwnerWithControlNamespace interface {
+	ResourceOwner
 	// control namespace dictates where namespaced objects should belong to
 	GetControlNamespace() string
 }
 
 type ResourceBuilders func(parent ResourceOwner, object interface{}) []ResourceBuilder
 type ResourceBuilder func() (runtime.Object, DesiredState, error)
+type ResourceTranslate func(runtime.Object) (parent ResourceOwner, config interface{})
+type PurgeTypesFunc func() []schema.GroupVersionKind
 
 type NativeReconciledComponent interface {
 	ResourceBuilders(parent ResourceOwner, object interface{}) []ResourceBuilder
@@ -93,11 +102,13 @@ func (d *DefaultReconciledComponent) PurgeTypes() []schema.GroupVersionKind {
 type NativeReconciler struct {
 	*GenericResourceReconciler
 	client.Client
-	scheme              *runtime.Scheme
-	reconciledComponent NativeReconciledComponent
-	configTranslate     func(runtime.Object) (parent ResourceOwner, config interface{})
-	componentName       string
-	setControllerRef    bool
+	scheme                 *runtime.Scheme
+	restMapper             meta.RESTMapper
+	reconciledComponent    NativeReconciledComponent
+	configTranslate        ResourceTranslate
+	componentName          string
+	setControllerRef       bool
+	reconciledObjectStates map[reconciledObjectState][]runtime.Object
 }
 
 type NativeReconcilerOpt func(*NativeReconciler)
@@ -114,6 +125,12 @@ func NativeReconcilerSetControllerRef() NativeReconcilerOpt {
 	}
 }
 
+func NativeReconcilerSetRESTMapper(mapper meta.RESTMapper) NativeReconcilerOpt {
+	return func(r *NativeReconciler) {
+		r.restMapper = mapper
+	}
+}
+
 func NewNativeReconciler(
 	componentName string,
 	rec *GenericResourceReconciler,
@@ -129,6 +146,8 @@ func NewNativeReconciler(
 		componentName:             componentName,
 	}
 
+	reconciler.initReconciledObjectStates()
+
 	for _, opt := range opts {
 		opt(reconciler)
 	}
@@ -159,11 +178,12 @@ func (rec *NativeReconciler) Reconcile(owner runtime.Object) (*reconcile.Result,
 		if err != nil {
 			combinedResult.CombineErr(err)
 		} else {
-			objectMeta, err := rec.addAnnotation(o, componentID)
+			objectMeta, err := rec.addComponentIDAnnotation(o, componentID)
 			if err != nil {
 				combinedResult.CombineErr(err)
 				continue
 			}
+			rec.addRelatedToAnnotation(objectMeta, ownerMeta)
 			if rec.setControllerRef {
 				isCrd := false
 				switch o.(type) {
@@ -173,9 +193,12 @@ func (rec *NativeReconciler) Reconcile(owner runtime.Object) (*reconcile.Result,
 					isCrd = true
 				}
 				if !isCrd {
-					if err := controllerutil.SetControllerReference(ownerMeta, objectMeta, rec.scheme); err != nil {
-						combinedResult.CombineErr(err)
-						continue
+					// namespaced resource can only own resources in the same namespace
+					if ownerMeta.GetNamespace() == "" || ownerMeta.GetNamespace() == objectMeta.GetNamespace() {
+						if err := controllerutil.SetControllerReference(ownerMeta, objectMeta, rec.scheme); err != nil {
+							combinedResult.CombineErr(err)
+							continue
+						}
 					}
 				}
 			}
@@ -187,6 +210,12 @@ func (rec *NativeReconciler) Reconcile(owner runtime.Object) (*reconcile.Result,
 					continue
 				}
 				excludeFromPurge[resourceID] = true
+
+				s := ReconciledObjectStatePresent
+				if state == StateAbsent {
+					s = ReconciledObjectStateAbsent
+				}
+				rec.addReconciledObjectState(s, o.DeepCopyObject())
 			}
 			combinedResult.Combine(result, err)
 		}
@@ -260,12 +289,42 @@ func (rec *NativeReconciler) generateResourceID(resource runtime.Object) (string
 	return strings.Join(identifiers, "-"), nil
 }
 
+func (rec *NativeReconciler) gvkExists(gvk schema.GroupVersionKind) bool {
+	if rec.restMapper == nil {
+		return true
+	}
+
+	mappings, err := rec.restMapper.RESTMappings(gvk.GroupKind(), gvk.Version)
+	if apimeta.IsNoMatchError(err) {
+		return false
+	}
+	if err != nil {
+		return true
+	}
+
+	for _, m := range mappings {
+		if gvk == m.GroupVersionKind {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (rec *NativeReconciler) purge(excluded map[string]bool, componentId string) error {
 	var allErr error
 	for _, gvk := range rec.reconciledComponent.PurgeTypes() {
+		rec.Log.V(2).Info("purging GVK", "gvk", gvk)
+		if !rec.gvkExists(gvk) {
+			continue
+		}
 		objects := &unstructured.UnstructuredList{}
 		objects.SetGroupVersionKind(gvk)
 		err := rec.List(context.TODO(), objects)
+		if apimeta.IsNoMatchError(err) {
+			// skip unknown GVKs
+			continue
+		}
 		if err != nil {
 			rec.Log.Error(err, "failed list objects to prune",
 				"groupversion", gvk.GroupVersion().String(),
@@ -286,23 +345,54 @@ func (rec *NativeReconciler) purge(excluded map[string]bool, componentId string)
 			if excluded[resourceID] {
 				continue
 			}
-			if objectMeta.GetAnnotations() != nil && objectMeta.GetAnnotations()[BanzaiCloudManagedComponent] == componentId {
+			if objectMeta.GetAnnotations()[types.BanzaiCloudManagedComponent] == componentId {
 				rec.Log.Info("pruning unmmanaged resource",
 					"name", objectMeta.GetName(),
 					"namespace", objectMeta.GetNamespace(),
 					"group", gvk.Group,
 					"version", gvk.Version,
 					"listKind", gvk.Kind)
-				if err := rec.Client.Delete(context.TODO(), &o); err != nil {
+				if err := rec.Client.Delete(context.TODO(), &o); err != nil && !k8serrors.IsNotFound(err) {
 					allErr = errors.Combine(allErr, err)
+				} else {
+					rec.addReconciledObjectState(ReconciledObjectStatePurged, o.DeepCopy())
 				}
 			}
 		}
 	}
 	return allErr
 }
 
-func (rec *NativeReconciler) addAnnotation(o runtime.Object, componentId string) (metav1.Object, error) {
+type reconciledObjectState string
+
+const (
+	ReconciledObjectStateAbsent  reconciledObjectState = "Absent"
+	ReconciledObjectStatePresent reconciledObjectState = "Present"
+	ReconciledObjectStatePurged  reconciledObjectState = "Purged"
+)
+
+func (rec *NativeReconciler) initReconciledObjectStates() {
+	rec.reconciledObjectStates = make(map[reconciledObjectState][]runtime.Object)
+}
+
+func (rec *NativeReconciler) addReconciledObjectState(state reconciledObjectState, o runtime.Object) {
+	rec.reconciledObjectStates[state] = append(rec.reconciledObjectStates[state], o)
+}
+
+func (rec *NativeReconciler) GetReconciledObjectWithState(state reconciledObjectState) []runtime.Object {
+	return rec.reconciledObjectStates[state]
+}
+
+func (rec *NativeReconciler) addRelatedToAnnotation(objectMeta, ownerMeta metav1.Object) {
+	annotations := objectMeta.GetAnnotations()
+	if annotations == nil {
+		annotations = make(map[string]string)
+	}
+	annotations[types.BanzaiCloudRelatedTo] = utils.ObjectKeyFromObjectMeta(ownerMeta).String()
+	objectMeta.SetAnnotations(annotations)
+}
+
+func (rec *NativeReconciler) addComponentIDAnnotation(o runtime.Object, componentId string) (metav1.Object, error) {
 	objectMeta, err := meta.Accessor(o)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to access object metadata")
@@ -311,14 +401,14 @@ func (rec *NativeReconciler) addAnnotation(o runtime.Object, componentId string)
 	if annotations == nil {
 		annotations = make(map[string]string)
 	}
-	if currentComponentId, ok := annotations[BanzaiCloudManagedComponent]; ok {
+	if currentComponentId, ok := annotations[types.BanzaiCloudManagedComponent]; ok {
 		if currentComponentId != componentId {
 			return nil, errors.Errorf(
 				"object actual component id `%s` is different from the one defined by the component `%s`",
 				currentComponentId, componentId)
 		}
 	} else {
-		annotations[BanzaiCloudManagedComponent] = componentId
+		annotations[types.BanzaiCloudManagedComponent] = componentId
 		objectMeta.SetAnnotations(annotations)
 	}
 	return objectMeta, nil