multicluster changes for kcp

Author: mjudeikis

.gitignore

@@ -23,3 +23,5 @@ go.work.sum
 
 # env file
 .env
+hack/tools/apigen
+bin/*

Makefile

@@ -96,8 +96,8 @@ build: manifests generate fmt vet ## Build manager binary.
 	go build -o bin/manager cmd/main.go
 
 .PHONY: run
-run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./cmd/main.go
+run: manifests generate kcp-generate fmt vet ## Run a controller from your host.
+	go run ./cmd/main.go --server=$$(kubectl get apiexport apis.contrib.kcp.io -o jsonpath="{.status.virtualWorkspaces[0].url}")
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
@@ -222,3 +222,30 @@ mv $(1) $(1)-$(3) ;\
 } ;\
 ln -sf $(1)-$(3) $(1)
 endef
+
+# MUTLICLUSTER-PROVIDER KCP SPECIFIC:
+TOOLS_DIR=hack/tools
+ARCH := $(shell go env GOARCH)
+OS := $(shell go env GOOS)
+
+KCP_APIGEN_BIN := apigen
+KCP_APIGEN_VERSION ?= 0.27.0-rc.1
+# Construct the download URL.
+DOWNLOAD_URL = https://github.com/kcp-dev/kcp/releases/download/v$(KCP_APIGEN_VERSION)/apigen_$(KCP_APIGEN_VERSION)_$(OS)_$(ARCH).tar.gz
+KCP_APIGEN_GEN := $(TOOLS_DIR)/$(KCP_APIGEN_BIN)
+export KCP_APIGEN_GEN # so hack scripts can use it
+
+# TODO: move to binary to avoid depeendencies
+$(KCP_APIGEN_GEN):
+# Create a temporary directory for download and extraction
+	TMP_DIR=$$(mktemp -d) ; \
+		curl -L -o $$TMP_DIR/apigen.tar.gz $(DOWNLOAD_URL) ; \
+		tar -xzf $$TMP_DIR/apigen.tar.gz -C $$TMP_DIR ; \
+		mv $$TMP_DIR/bin/apigen $(KCP_APIGEN_GEN) ; \
+		rm -rf $$TMP_DIR
+
+kcp-tools: $(KCP_APIGEN_GEN) 
+.PHONY: kcp-tools
+
+kcp-generate:
+	$(TOOLS_DIR)/apigen --input-dir ./config/crd/bases --output-dir ./config/kcp    

README.md

@@ -1,2 +1,79 @@
 # multicluster-provider-kubebuilder-example
-A full kcp multicluster-provider example using kubebuilder
+
+A full kcp multicluster-provider example using kubebuilder.
+
+This example demonstrates how to lift&shift a kubebuilder project to a multicluster-provider project for kcp.
+
+There should be 2 commits in this repository at all times:
+
+1. The initial commit with the kubebuilder project.
+
+Skeleton created by running [kubebuilder](https://book.kubebuilder.io/quick-start.html) with the following command:
+
+```sh
+$ kubebuilder init --domain contrib.kcp.io --repo github.com/kcp-dev/multicluster-provider/examples/crd
+$ kubebuilder create api --group apis --version v1alpha1 --kind Application
+```
+
+2. The commit with the kubebuilder project lifted&shifted to a multicluster-provider project to work with kcp.
+
+Please refer to main repositories for more information:
+
+- [multicluster-provider for kcp](https://github.com/kcp-dev/multicluster-provider)
+- [multicluster-runtime](https://github.com/multicluster-runtime/multicluster-runtime)
+- [kcp](https://github.com/kcp-dev/kcp)
+
+
+## Customizations & Run
+
+### Prerequisites
+
+- [kcp](https://github.com/kcp-dev/kcp) installed and running
+- [kcp krew plugins](https://docs.kcp.io/kcp/v0.26/setup/kubectl-plugin/) installed
+
+1. Make file targets to install kcp specific tooling and generate kcp specific manifests:
+
+```sh
+# install tooling
+make kcp-tools
+# generate kcp manifests
+make kcp-generate
+```
+
+After than content of `cmd/main.go` was updated with multicluster extension.
+
+It can be tested by applying the necessary manifests from the respective folder while connected to the `root` workspace of a kcp instance:
+
+export KUBECONFIG=admin.kubeconfig
+
+```sh
+$ kubectl ws create provider --enter
+$ kubectl apply -f ./config/kcp/apiresourceschema-applications.apis.contrib.kcp.io.yaml
+$ kubectl apply -f ./config/kcp/apiexport-apis.contrib.kcp.io.yaml
+
+# Consumer 1
+$ kubectl ws use :root
+$ kubectl ws create examples-crd-multicluster-1 --enter
+$ kubectl kcp bind apiexport root:provider:apis.contrib.kcp.io 
+$ kubectl apply -f ./config/samples/apis_v1alpha1_application.yaml
+
+# Consumer 2
+$ kubectl ws use :root
+$ kubectl ws create examples-crd-multicluster-2 --enter
+$ kubectl kcp bind apiexport root:provider:apis.contrib.kcp.io 
+$ kubectl apply -f ./config/samples/apis_v1alpha1_application.yaml
+```
+
+Then, start the example controller by passing the virtual workspace URL to it:
+
+
+```sh
+$ kubectl ws use :root:provider
+$ go run . --server=$(kubectl get apiexport apis.contrib.kcp.io -o jsonpath="{.status.virtualWorkspaces[0].url}")
+```
+
+Observe the controller reconciling the ` application-sample` Aplication in the workspaces:
+
+```sh
+2025-03-11T13:04:52+02:00       INFO    Reconciling Application {"controller": "kcp-applications-controller", "controllerGroup": "apis.contrib.kcp.io", "controllerKind": "Application", "reconcileID": "babfc696-50cc-4851-ab35-d1d956a6c120", "cluster": "1058d5hgzdd3ask6"}
+```

bin/.gitkeep

@@ -17,27 +17,39 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"os"
 	"path/filepath"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
+
+	apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1"
+	"github.com/kcp-dev/multicluster-provider/virtualworkspace"
+	mcbuilder "github.com/multicluster-runtime/multicluster-runtime/pkg/builder"
+	mcmanager "github.com/multicluster-runtime/multicluster-runtime/pkg/manager"
+	mcreconcile "github.com/multicluster-runtime/multicluster-runtime/pkg/reconcile"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
-	apisv1alpha1 "github.com/kcp-dev/multicluster-provider/examples/crd/api/v1alpha1"
+	applicationapisv1alpha1 "github.com/kcp-dev/multicluster-provider/examples/crd/api/v1alpha1"
 	"github.com/kcp-dev/multicluster-provider/examples/crd/internal/controller"
 	// +kubebuilder:scaffold:imports
 )
@@ -49,7 +61,8 @@ var (
 
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-
+	// MULTICLUSTER: This is where it differ from the default scaffold.
+	utilruntime.Must(applicationapisv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(apisv1alpha1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
@@ -63,6 +76,7 @@ func main() {
 	var probeAddr string
 	var secureMetrics bool
 	var enableHTTP2 bool
+	var server string
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -81,6 +95,8 @@ func main() {
 	flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
+	// MULTICLUSTER: This is where it differ from the default scaffold.
+	flag.StringVar(&server, "server", "", "Override for kubeconfig server URL")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -178,7 +194,25 @@ func main() {
 		})
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	// MULTICLUSTER: This is where it differ from the default scaffold.
+	ctx := signals.SetupSignalHandler()
+
+	cfg := ctrl.GetConfigOrDie()
+	cfg = rest.CopyConfig(cfg)
+	if server != "" {
+		cfg.Host = server
+	}
+
+	var err error
+	provider, err := virtualworkspace.New(cfg, &apisv1alpha1.APIBinding{}, virtualworkspace.Options{
+		Scheme: scheme,
+	})
+	if err != nil {
+		setupLog.Error(err, "unable to construct cluster provider")
+		os.Exit(1)
+	}
+
+	mgr, err := mcmanager.New(cfg, provider, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsServerOptions,
 		WebhookServer:          webhookServer,
@@ -202,30 +236,48 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err = (&controller.ApplicationReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-	}).SetupWithManager(mgr); err != nil {
+	if err := mcbuilder.ControllerManagedBy(mgr).
+		Named("kcp-applications-controller").
+		For(&applicationapisv1alpha1.Application{}).
+		Complete(mcreconcile.Func(
+			func(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) {
+				log := log.FromContext(ctx).WithValues("cluster", req.ClusterName)
+				log.Info("Reconciling Application")
+
+				cl, err := mgr.GetCluster(ctx, req.ClusterName)
+				if err != nil {
+					return reconcile.Result{}, fmt.Errorf("failed to get cluster: %w", err)
+				}
+				client := cl.GetClient()
+
+				reconciler := &controller.ApplicationReconciler{
+					Client: client,
+					Scheme: cl.GetScheme(),
+				}
+				return reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: req.NamespacedName})
+			},
+		)); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Application")
 		os.Exit(1)
 	}
 	// +kubebuilder:scaffold:builder
 
-	if metricsCertWatcher != nil {
-		setupLog.Info("Adding metrics certificate watcher to manager")
-		if err := mgr.Add(metricsCertWatcher); err != nil {
-			setupLog.Error(err, "unable to add metrics certificate watcher to manager")
-			os.Exit(1)
-		}
-	}
-
-	if webhookCertWatcher != nil {
-		setupLog.Info("Adding webhook certificate watcher to manager")
-		if err := mgr.Add(webhookCertWatcher); err != nil {
-			setupLog.Error(err, "unable to add webhook certificate watcher to manager")
-			os.Exit(1)
-		}
-	}
+	// TODO(mjudeikis): This needs to be implemented in mcmanager.
+	// if metricsCertWatcher != nil {
+	//	setupLog.Info("Adding metrics certificate watcher to manager")
+	//	if err := mgr.Add(metricsCertWatcher); err != nil {
+	//		setupLog.Error(err, "unable to add metrics certificate watcher to manager")
+	//		os.Exit(1)
+	//	}
+	// }
+	//
+	// if webhookCertWatcher != nil {
+	//	setupLog.Info("Adding webhook certificate watcher to manager")
+	//	if err := mgr.Add(webhookCertWatcher); err != nil {
+	//		setupLog.Error(err, "unable to add webhook certificate watcher to manager")
+	//		os.Exit(1)
+	//	}
+	// }
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
@@ -237,7 +289,18 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if provider != nil {
+		setupLog.Info("Starting provider")
+		go func() {
+			if err := provider.Run(ctx, mgr); err != nil {
+				setupLog.Error(err, "unable to run provider")
+				os.Exit(1)
+			}
+		}()
+	}
+
+	setupLog.Info("starting manager", "server", server)
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

bin/controller-gen

@@ -0,0 +1,9 @@
+apiVersion: apis.kcp.io/v1alpha1
+kind: APIExport
+metadata:
+  creationTimestamp: null
+  name: apis.contrib.kcp.io
+spec:
+  latestResourceSchemas:
+  - v250316-6eed6de.applications.apis.contrib.kcp.io
+status: {}

bin/controller-gen-v0.17.2

@@ -0,0 +1,51 @@
+apiVersion: apis.kcp.io/v1alpha1
+kind: APIResourceSchema
+metadata:
+  creationTimestamp: null
+  name: v250316-6eed6de.applications.apis.contrib.kcp.io
+spec:
+  group: apis.contrib.kcp.io
+  names:
+    kind: Application
+    listKind: ApplicationList
+    plural: applications
+    singular: application
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      description: Application is the Schema for the applications API.
+      properties:
+        apiVersion:
+          description: |-
+            APIVersion defines the versioned schema of this representation of an object.
+            Servers should convert recognized schemas to the latest internal value, and
+            may reject unrecognized values.
+            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+          type: string
+        kind:
+          description: |-
+            Kind is a string value representing the REST resource this object represents.
+            Servers may infer this from the endpoint the client submits requests to.
+            Cannot be updated.
+            In CamelCase.
+            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: ApplicationSpec defines the desired state of Application.
+          properties:
+            foo:
+              description: Foo is an example field of Application. Edit application_types.go
+                to remove/update
+              type: string
+          type: object
+        status:
+          description: ApplicationStatus defines the observed state of Application.
+          type: object
+      type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}