Pod eviction controller (#117)

Co-authored-by: Gerrit <[email protected]>

Author: ostempel

.github/workflows/docker.yaml

@@ -15,6 +15,7 @@ env:
   REGISTRY: ghcr.io
   PLUGIN_IMAGE_NAME: ${{ github.repository }}
   PROVISIONER_IMAGE_NAME: ${{ github.repository }}-provisioner
+  CONTROLLER_IMAGE_NAME: ${{ github.repository }}-controller
 
 jobs:
   lint:
@@ -82,25 +83,28 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
 
       - name: Build
         env:
           GOOS: ${{ matrix.os }}
           GOARCH: ${{ matrix.arch }}
           GOARM: ${{ matrix.arch == 'arm' && '7' || '' }}
         run: |
-          make lvmplugin provisioner
+          make lvmplugin provisioner controller
 
       - uses: actions/upload-artifact@v4
         with:
           name: lvmplugin-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.arch == 'arm' && '-v7' || '' }}
           path: bin
-
       - uses: actions/upload-artifact@v4
         with:
           name: provisioner-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.arch == 'arm' && '-v7' || '' }}
           path: bin
+      - uses: actions/upload-artifact@v4
+        with:
+          name: controller-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.arch == 'arm' && '-v7' || '' }}
+          path: bin
 
   docker-build:
     runs-on: ubuntu-latest
@@ -131,20 +135,24 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-
       - name: Download plugin binaries
         uses: actions/download-artifact@v4
         with:
           pattern: lvmplugin-*
           path: bin
           merge-multiple: true
-
       - name: Download provisioner binaries
         uses: actions/download-artifact@v4
         with:
           pattern: provisioner-*
           path: bin
           merge-multiple: true
+      - name: Download controller binaries
+        uses: actions/download-artifact@v4
+        with:
+          pattern: controller-*
+          path: bin
+          merge-multiple: true
 
       - name: Build and push plugin image
         if: ${{ env.DOCKER_REGISTRY_TOKEN != '' }}
@@ -167,3 +175,14 @@ jobs:
           tags: ${{ env.REGISTRY }}/${{ env.PROVISIONER_IMAGE_NAME }}:${{ env.tag }}
           file: cmd/provisioner/Dockerfile
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+
+      - name: Build and push controller image
+        if: ${{ env.DOCKER_REGISTRY_TOKEN != '' }}
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          sbom: true
+          tags: ${{ env.REGISTRY }}/${{ env.CONTROLLER_IMAGE_NAME }}:${{ env.tag }}
+          file: cmd/controller/Dockerfile
+          platforms: linux/amd64,linux/arm64,linux/arm/v7

.gitignore

@@ -5,3 +5,31 @@ loop*
 # editor specific files
 .idea
 .vscode
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin/*
+Dockerfile.cross
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# Kubernetes Generated files - skip generated files, except for vendored files
+!vendor/**/zz_generated.*
+
+# editor and IDE paraphernalia
+.idea
+.vscode
+*.swp
+*.swo
+*~

Makefile

@@ -1,21 +1,31 @@
 GOOS ?= linux
 GOARCH ?= amd64
-GOARM ?= 
+GOARM ?=
 CGO_ENABLED ?= 0
 TAGS := -tags 'osusergo netgo static_build'
 
 PLATFORM := $(GOOS)/$(GOARCH)$(if $(GOARM),/v$(GOARM))
 BINARY_LVMPLUGIN := $(PLATFORM)/lvmplugin
 BINARY_PROVISIONER:= $(PLATFORM)/provisioner
+BINARY_CONTROLLER:= $(PLATFORM)/controller
+
+GO111MODULE := on
+KUBECONFIG := $(shell pwd)/.kubeconfig
+HELM_REPO := "https://helm.metal-stack.io"
 
 SHA := $(shell git rev-parse --short=8 HEAD)
 GITVERSION := $(shell git describe --long --all)
-BUILDDATE := $(shell date --rfc-3339=seconds)
+# gnu date format iso-8601 is parsable with Go RFC3339
+BUILDDATE := $(shell date --iso-8601=seconds)
 VERSION := $(or ${VERSION},$(shell git describe --tags --exact-match 2> /dev/null || git symbolic-ref -q --short HEAD || git rev-parse --short HEAD))
 
-GO111MODULE := on
-KUBECONFIG := $(shell pwd)/.kubeconfig
-HELM_REPO := "https://helm.metal-stack.io"
+CONTROLLER_TOOLS_VERSION ?= v0.18.0
+LOCALBIN ?= $(shell pwd)/bin
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+ENVTEST ?= $(LOCALBIN)/setup-envtest
+
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
 
 ifeq ($(CI),true)
   DOCKER_TTY_ARG=
@@ -70,7 +80,7 @@ build-provisioner: provisioner
 	docker build -t csi-driver-lvm-provisioner -f cmd/provisioner/Dockerfile .
 
 /dev/loop%:
-	@fallocate --length 2G loop$*.img
+	@fallocate --length 4G loop$*.img
 ifndef GITHUB_ACTIONS
 	@sudo mknod $@ b 7 $*
 endif
@@ -96,25 +106,87 @@ kind:
 			--kubeconfig $(KUBECONFIG); fi
 	@kind --name csi-driver-lvm load docker-image csi-driver-lvm
 	@kind --name csi-driver-lvm load docker-image csi-driver-lvm-provisioner
+	@kind --name csi-driver-lvm load docker-image csi-driver-lvm-controller
 
 .PHONY: rm-kind
 rm-kind:
 	@kind delete cluster --name csi-driver-lvm
 
 RERUN ?= 1
 .PHONY: test
-test: build-plugin build-provisioner /dev/loop100 /dev/loop101 kind
+test: build-plugin build-provisioner build-controller /dev/loop100 /dev/loop101 kind
 	@cd tests && docker build -t csi-bats . && cd -
 	@touch $(KUBECONFIG)
 	@for i in {1..$(RERUN)}; do \
 	docker run -i$(DOCKER_TTY_ARG) \
 		-e HELM_REPO=$(HELM_REPO) \
 		-v "$(KUBECONFIG):/root/.kube/config" \
 		-v "$(PWD)/tests:/code" \
+		-v "$(PWD)/config:/config" \
 		--network host \
 		csi-bats \
 		--verbose-run --trace --timing bats/test.bats ; \
 	done
 
 .PHONY: test-cleanup
 test-cleanup: rm-kind
+
+#!
+#! CONTROLLER
+#!
+
+.PHONY: controller
+controller: generate fmt #vet
+	go mod tidy
+	go build \
+		$(TAGS) \
+		-ldflags \
+		"$(LINKMODE)" \
+		-o bin/$(BINARY_CONTROLLER) \
+		./cmd/controller/main.go
+	cd bin/ && \
+	sha512sum $(BINARY_CONTROLLER) > $(BINARY_CONTROLLER).sha512
+
+.PHONY: build-controller
+build-controller: controller
+	docker build -t csi-driver-lvm-controller -f cmd/controller/Dockerfile .
+
+.PHONY: manifests
+manifests: controller-gen 
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+
+deploy: manifests
+	cd config/manager && kustomize edit set image controller=csi-driver-lvm-controller
+	kustomize build config/default | kubectl apply -f -
+
+.PHONY: undeploy
+undeploy: 
+	kustomize build config/default | kubectl delete -f -
+
+.PHONY: generate
+generate: controller-gen manifests
+	go generate ./...
+	$(CONTROLLER_GEN) object paths="./..."
+
+.PHONY: fmt
+fmt:
+	go fmt ./...
+
+.PHONY: vet
+vet:
+	go vet ./...
+
+# .PHONY: test
+# test: manifests generate fmt vet setup-envtest ## Run tests.
+# 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN)
+$(CONTROLLER_GEN): $(LOCALBIN)
+	test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \
+	GOOS= GOARCH= GOARM= GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
+
+.PHONY: setup-envtest 
+setup-envtest: $(ENVTEST)
+$(ENVTEST): $(LOCALBIN)
+	test -s $(LOCALBIN)/setup-envtest || GOOS= GOARCH= GOARM= GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest

README.md

@@ -12,6 +12,17 @@ This CSI driver is derived from [csi-driver-host-path](https://github.com/kubern
 
 For the special case of block volumes, the filesystem-expansion has to be performed by the app using the block device
 
+## Automatic PVC Deletion on Pod Eviction
+
+The persistent volumes created by this CSI driver are strictly node-affine to the node on which the pod was scheduled. This is intentional and prevents pods from starting without the LV data, which resides only on the specific node in the Kubernetes cluster. 
+
+Consequently, if a pod is evicted (potentially due to cluster autoscaling or updates to the worker node), the pod may become stuck. In certain scenarios, it's acceptable for the pod to start on another node, despite the potential for data loss. The csi-driver-lvm-controller can capture these events and automatically delete the PVC without requiring manual intervention by an operator.
+
+To use this functionality, the following is needed:
+
+- This only works on `StatefulSet`s with volumeClaimTemplates and volume references to the `csi-driver-lvm` storage class
+- In addition to that, the `Pod` or `PersistentVolumeClaim` managed by the `StatefulSet` needs the annotation: `metal-stack.io/csi-driver-lvm.is-eviction-allowed: true`
+
 ## Installation ##
 
 **Helm charts for installation are located in a separate repository called [helm-charts](https://github.com/metal-stack/helm-charts). If you would like to contribute to the helm chart, please raise an issue or pull request there.**
@@ -65,6 +76,7 @@ You can create these loop devices like this:
 ```bash
 for i in 100 101; do fallocate -l 1G loop${i}.img ; sudo losetup /dev/loop${i} loop${i}.img; done
 sudo losetup -a
+# https://github.com/util-linux/util-linux/issues/3197
 # use this for recreation or cleanup
 # for i in 100 101; do sudo losetup -d /dev/loop${i}; rm -f loop${i}.img; done
 ```

cmd/controller/Dockerfile

@@ -0,0 +1,8 @@
+FROM gcr.io/distroless/static:nonroot 
+ARG TARGETPLATFORM
+LABEL maintainer="metal-stack authors <[email protected]>"
+
+WORKDIR /
+COPY --chmod=755 bin/${TARGETPLATFORM}/controller /controller
+USER 65532:65532
+ENTRYPOINT ["/controller"]

cmd/controller/main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/metal-stack/csi-driver-lvm/pkg/controller"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	// +kubebuilder:scaffold:imports
+)
+
+var (
+	scheme   = runtime.NewScheme()
+	setupLog = ctrl.Log.WithName("setup")
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+
+	// +kubebuilder:scaffold:scheme
+}
+
+// nolint:gocyclo
+func main() {
+	var (
+		provisionerName      string
+		logLevel             string
+		metricsAddr          string
+		enableLeaderElection bool
+		probeAddr            string
+	)
+	flag.StringVar(&provisionerName, "provisioner-name", "lvm.csi.metal-stack.io", "The provisioner name of the csi-driver.")
+	flag.StringVar(&logLevel, "log-level", "info", "The log level of the application.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
+		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
+	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
+		"Enable leader election for controller manager. "+
+			"Enabling this will ensure there is only one active controller manager.")
+
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(flag.CommandLine)
+	flag.Parse()
+
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme: scheme,
+		Metrics: server.Options{
+			BindAddress: metricsAddr,
+		},
+		HealthProbeBindAddress: probeAddr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "814f7540.csi-driver-lvm-controller",
+	})
+	if err != nil {
+		setupLog.Error(err, "unable to start manager")
+		os.Exit(1)
+	}
+
+	// +kubebuilder:scaffold:builder
+
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
+	reconciler := controller.New(mgr.GetClient(), mgr.GetScheme(), ctrl.Log.WithName("CsiDriverLvmReconciler"), controller.Config{ProvisionerName: provisionerName})
+
+	if err := reconciler.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "CsiDriverLvm")
+		os.Exit(1)
+	}
+
+	setupLog.Info("starting manager")
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		setupLog.Error(err, "problem running manager")
+		os.Exit(1)
+	}
+}

cmd/lvmplugin/Dockerfile

@@ -3,6 +3,6 @@ ARG TARGETPLATFORM
 LABEL maintainer="metal-stack authors <[email protected]>"
 
 RUN apk add lvm2 lvm2-extra e2fsprogs e2fsprogs-extra smartmontools nvme-cli util-linux device-mapper xfsprogs xfsprogs-extra
-COPY bin/${TARGETPLATFORM}/lvmplugin /lvmplugin
+COPY --chmod=755 bin/${TARGETPLATFORM}/lvmplugin /lvmplugin
 USER root
 ENTRYPOINT ["/lvmplugin"]

cmd/provisioner/Dockerfile

@@ -3,6 +3,6 @@ ARG TARGETPLATFORM
 LABEL maintainer="metal-stack authors <[email protected]>"
 
 RUN apk add lvm2 lvm2-extra e2fsprogs e2fsprogs-extra xfsprogs-extra smartmontools nvme-cli util-linux device-mapper
-COPY bin/${TARGETPLATFORM}/provisioner /csi-lvmplugin-provisioner
+COPY --chmod=755 bin/${TARGETPLATFORM}/provisioner /csi-lvmplugin-provisioner
 USER root
 ENTRYPOINT ["/csi-lvmplugin-provisioner"]