Merge pull request #97 from cloudscale-ch/release/3.5.7-prerelease

fix race conditions on stage/unstage volume

Author: mweibel

.github/workflows/release-chart.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 'release/[0-9]+.[0-9]+.*'
 
 jobs:
   release:

CHANGELOG.md

@@ -1,5 +1,12 @@
 ## unreleased
-* Minimum supported Kubernetes version is now 1.28.
+* The minimum supported Kubernetes version is now 1.28.
+* Upgrade all sidecars to latest version
+  * Please note the RBAC changes as a result of this upgrade
+* Fix race conditions on stage/unstage/publish/unpublish volume
+  * Add volume locks to avoid concurrent operations on the same volume
+  * Add detection if symlinks don't add up to the right volume
+  * Adjust udevadm calls & reorder them and put them behind a mutex to avoid a potential race condition
+* Add `--log-level=(trace|debug|info|warn|error)` flag to customize log level
 
 ## v3.5.6 - 2024.04.18
 * ~~Add support for Kubernetes 1.30~~

Makefile

@@ -15,7 +15,7 @@ VERSION ?= $(shell cat VERSION)
 CHART_VERSION ?= $(shell awk '/^version:/ {print $$2}' charts/csi-cloudscale/Chart.yaml)
 DOCKER_REPO ?= quay.io/cloudscalech/cloudscale-csi-plugin
 
-all: check-unused test
+all: lint check-unused test
 
 publish: build push clean
 
@@ -60,12 +60,12 @@ check-unused:
 .PHONY: test
 test:
 	@echo "==> Testing all packages"
-	@GO111MODULE=on go test -v ./... $(TEST_ARGS)
+	@GO111MODULE=on go test -race -v ./... $(TEST_ARGS)
 
 .PHONY: test-integration
 test-integration:
 	@echo "==> Started integration tests"
-	@env GO111MODULE=on go test -count 1 -v $(TESTARGS) -tags integration -timeout 20m ./test/...
+	@env GO111MODULE=on go test -race -count 1 -v $(TESTARGS) -tags integration -timeout 20m ./test/...
 
 .PHONY: build
 build: compile
@@ -74,10 +74,10 @@ build: compile
 
 .PHONY: push
 push:
-ifeq ($(DOCKER_REPO),cloudscalech/cloudscale-csi-plugin)
-  ifneq ($(BRANCH),master)
+ifeq ($(DOCKER_REPO),quay.io/cloudscalech/cloudscale-csi-plugin)
+  ifeq ($(filter master,$(BRANCH))$(filter release/%,$(BRANCH)),)
     ifneq ($(VERSION),dev)
-	  $(error "Only the `dev` tag can be published from non-master branches")
+	  $(error "Only the `dev` tag can be published from non-master/non-release branches")
     endif
   endif
 endif
@@ -93,3 +93,15 @@ clean:
 .PHONY: helm-template
 helm-template:
 	@helm template csi-cloudscale -n kube-system --set nameOverride=csi-cloudscale ./charts/csi-cloudscale
+
+## Development tooling
+
+fmt: ## Run go fmt against code.
+	go fmt ./...
+
+vet: ## Run go vet against code.
+	go vet ./...
+
+.PHONY: lint
+lint: fmt vet ## Combined target to run linters
+	go tool -modfile tool.mod golangci-lint run --timeout 2m0s ./...

README.md

@@ -61,7 +61,7 @@ secret `my-pvc-luks-key`.
 ## Releases
 
 The cloudscale.ch CSI plugin follows [semantic versioning](https://semver.org/).
-The current version is: **`v3.5.6`**.
+The current version is: **`v3.5.7-rc2`**.
 
 * Bug fixes will be released as a `PATCH` update.
 * New features (such as CSI spec bumps) will be released as a `MINOR` update.
@@ -89,14 +89,14 @@ We recommend using the latest cloudscale.ch CSI driver compatible with your Kube
 | 1.25               | v3.3.0                           | v3.5.6                           |
 | 1.26               | v3.3.0                           | v3.5.6                           |
 | 1.27               | v3.3.0                           | v3.5.6                           |
-| 1.28               | v3.3.0                           | v3.5.6                           |
-| 1.29               | v3.3.0                           | v3.5.6                           |
-| 1.30               | v3.3.0                           | v3.5.6                           |
-| 1.31               | v3.3.0                           | v3.5.6                           |
-| 1.32               | v3.3.0                           | v3.5.6                           |
-| 1.33               | v3.3.0                           | v3.5.6                           |
-| 1.34 [1]           | v3.3.0                           | v3.5.6                           |
-| 1.35               | v3.4.1                           | v3.5.6                           |
+| 1.28               | v3.3.0                           | v3.5.7-rc2                       |
+| 1.29               | v3.3.0                           | v3.5.7-rc2                       |
+| 1.30               | v3.3.0                           | v3.5.7-rc2                       |
+| 1.31               | v3.3.0                           | v3.5.7-rc2                       |
+| 1.32               | v3.3.0                           | v3.5.7-rc2                       |
+| 1.33               | v3.3.0                           | v3.5.7-rc2                       |
+| 1.34 [1]           | v3.3.0                           | v3.5.7-rc2                       |
+| 1.35               | v3.4.1                           | v3.5.7-rc2                       |
 
 [1] Prometheus `kubelet_volume_stats_*` metrics not available in 1.34.0 and 1.34.1 due to a 
     [bug in Kubelet](https://github.com/kubernetes/kubernetes/issues/133847). Fixed in `1.34.2`.
@@ -198,10 +198,10 @@ $ helm install -g -n kube-system --set controller.image.tag=dev --set node.image
 Before you continue, be sure to checkout to a [tagged
 release](https://github.com/cloudscale-ch/csi-cloudscale/releases). 
 Always use the [latest stable version](https://github.com/cloudscale-ch/csi-cloudscale/releases/latest) 
-For example, to use the latest stable version (`v3.5.6`) you can execute the following command:
+For example, to use the latest stable version (`v3.5.7-rc2`) you can execute the following command:
 
 ```
-$ kubectl apply -f https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-v3.5.6.yaml
+$ kubectl apply -f https://raw.githubusercontent.com/cloudscale-ch/csi-cloudscale/master/deploy/kubernetes/releases/csi-cloudscale-v3.5.7-rc2.yaml
 ```
 
 The storage classes `cloudscale-volume-ssd` and `cloudscale-volume-bulk` will be created. The 
@@ -421,15 +421,23 @@ $ git push origin
 
 After it's merged to master, [create a new Github
 release](https://github.com/cloudscale-ch/csi-cloudscale/releases/new) from
-master with the version `v3.5.6` and then publish a new docker build:
+master with the version `v3.5.7-rc2` and then publish a new docker build:
 
 ```
 $ git checkout master
 $ make publish
 ```
 
-This will create a binary with version `v3.5.6` and docker image pushed to
-`cloudscalech/cloudscale-csi-plugin:v3.5.6`
+This will create a binary with version `v3.5.7-rc2` and docker image pushed to
+`cloudscalech/cloudscale-csi-plugin:v3.5.7-rc2`
+
+### Release a pre-release version
+
+To release a new pre-release (or a patch version based on an existing tag) version, follow the following steps:
+
+1. create a new branch called `release/x.y` (or with the full version)
+2. push it to GitHub
+3. Follow the flow for creating a new release, making sure to set the appropriate versions.
 
 ## Contributing
 

VERSION

@@ -1 +1 @@
-v3.5.6
+v3.5.7-rc2

charts/csi-cloudscale/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: csi-cloudscale
 description: A Container Storage Interface Driver for cloudscale.ch volumes.
 type: application
-version: 1.3.6
-appVersion: "3.5.6"
+version: 1.3.7-rc2
+appVersion: "3.5.7-rc2"
 home: https://github.com/cloudscale-ch/csi-cloudscale
 sources:
   - https://github.com/cloudscale-ch/csi-cloudscale.git

charts/csi-cloudscale/templates/daemonset.yaml

@@ -15,7 +15,7 @@ spec:
         role: csi-cloudscale
     spec:
       priorityClassName: system-node-critical
-      serviceAccount: {{ include "csi-cloudscale.node-service-account-name" . }}
+      serviceAccountName: {{ include "csi-cloudscale.node-service-account-name" . }}
       hostNetwork: true
       containers:
         - name: csi-node-driver-registrar
@@ -53,6 +53,7 @@ spec:
           args :
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--url=$(CLOUDSCALE_API_URL)"
+            - "--log-level={{ .Values.node.logLevel }}"
           {{- with .Values.node.resources }}
           resources:
 {{ toYaml . | indent 12 }}

charts/csi-cloudscale/templates/rbac.yaml

@@ -18,7 +18,7 @@ rules:
     verbs: ["list", "watch", "create", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
-    verbs: ["get", "list"]
+    verbs: [ "get", "list", "watch", "update" ]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
     verbs: ["get", "list"]
@@ -40,9 +40,6 @@ rules:
   - apiGroups: [""]
     resources: ["persistentvolumes"]
     verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
@@ -64,12 +61,18 @@ rules:
   - apiGroups: [""]
     resources: ["persistentvolumeclaims"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
+    verbs: ["get", "list", "watch"]
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1

charts/csi-cloudscale/templates/statefulset.yaml

@@ -77,6 +77,7 @@ spec:
           args :
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--url=$(CLOUDSCALE_API_URL)"
+            - "--log-level={{ .Values.controller.logLevel }}"
           {{- with .Values.controller.resources }}
           resources:
 {{ toYaml . | indent 12 }}

charts/csi-cloudscale/values.yaml

@@ -84,9 +84,10 @@ controller:
   image:
     registry: quay.io
     repository: cloudscalech/cloudscale-csi-plugin
-    tag: v3.5.6
+    tag: v3.5.7-rc2
     pullPolicy: IfNotPresent
   serviceAccountName:
+  logLevel: info
   resources: {}
 #     limits:
 #      cpu: 100m
@@ -99,11 +100,12 @@ node:
   image:
     registry: quay.io
     repository: cloudscalech/cloudscale-csi-plugin
-    tag: v3.5.6
+    tag: v3.5.7-rc2
     pullPolicy: IfNotPresent
   nodeSelector: {}
   tolerations: []
   serviceAccountName:
+  logLevel: info
   resources: {}
 #     limits:
 #      cpu: 100m