Merge pull request #697 from red-hat-storage/sync_us--master

Syncing latest changes from upstream master for rook

.github/workflows/integration-test-helm-suite.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-keystone-auth-suite.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.28.4"]
+        kubernetes-versions: ["v1.26.15", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-mgr-suite.yaml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.30.0"]
+        kubernetes-versions: ["v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-multi-cluster-suite.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.30.0"]
+        kubernetes-versions: ["v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-object-suite.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-smoke-suite.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-upgrade-suite.yaml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-tests-on-release.yaml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.27.13", "v1.28.9", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.28.12", "v1.29.7", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -61,7 +61,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.27.13", "v1.28.9", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.28.12", "v1.29.7", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -102,7 +102,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.27.13", "v1.28.9", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.28.12", "v1.29.7", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -140,7 +140,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.27.13", "v1.28.9", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.28.12", "v1.29.7", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -178,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.27.13", "v1.28.9", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.28.12", "v1.29.7", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -219,7 +219,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.25.16", "v1.30.0"]
+        kubernetes-versions: ["v1.26.15", "v1.31.0"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.mergify.yml

@@ -274,17 +274,17 @@ pull_request_rules:
       - "check-success=canary-tests / encryption-pvc-kms-ibm-kp (quay.io/ceph/ceph:v18)"
       - "check-success=canary-tests / multus-cluster-network (quay.io/ceph/ceph:v18)"
       - "check-success=canary-tests / csi-hostnetwork-disabled (quay.io/ceph/ceph:v18)"
-      - "check-success=TestCephSmokeSuite (v1.25.16)"
-      - "check-success=TestCephSmokeSuite (v1.30.0)"
-      - "check-success=TestCephHelmSuite (v1.25.16)"
-      - "check-success=TestCephHelmSuite (v1.30.0)"
-      - "check-success=TestCephMultiClusterDeploySuite (v1.30.0)"
-      - "check-success=TestCephObjectSuite (v1.25.16)"
-      - "check-success=TestCephObjectSuite (v1.30.0)"
-      - "check-success=TestCephUpgradeSuite (v1.25.16)"
-      - "check-success=TestCephUpgradeSuite (v1.30.0)"
-      - "check-success=TestHelmUpgradeSuite (v1.25.16)"
-      - "check-success=TestHelmUpgradeSuite (v1.30.0)"
+      - "check-success=TestCephSmokeSuite (v1.26.15)"
+      - "check-success=TestCephSmokeSuite (v1.31.0)"
+      - "check-success=TestCephHelmSuite (v1.26.15)"
+      - "check-success=TestCephHelmSuite (v1.31.0)"
+      - "check-success=TestCephMultiClusterDeploySuite (v1.31.0)"
+      - "check-success=TestCephObjectSuite (v1.26.15)"
+      - "check-success=TestCephObjectSuite (v1.31.0)"
+      - "check-success=TestCephUpgradeSuite (v1.26.15)"
+      - "check-success=TestCephUpgradeSuite (v1.31.0)"
+      - "check-success=TestHelmUpgradeSuite (v1.26.15)"
+      - "check-success=TestHelmUpgradeSuite (v1.31.0)"
     actions:
       merge:
         method: merge

Documentation/Getting-Started/Prerequisites/prerequisites.md

@@ -7,7 +7,7 @@ and Rook is granted the required privileges (see below for more information).
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.25** through **v1.30** are supported.
+Kubernetes versions **v1.26** through **v1.31** are supported.
 
 ## CPU Architecture
 

Documentation/Getting-Started/quickstart.md

@@ -12,7 +12,7 @@ This guide will walk through the basic setup of a Ceph cluster and enable K8s ap
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.25** through **v1.30** are supported.
+Kubernetes versions **v1.26** through **v1.31** are supported.
 
 ## CPU Architecture
 

Documentation/Upgrade/ceph-upgrade.md

@@ -24,14 +24,11 @@ until all the daemons have been updated.
 
 ## Supported Versions
 
-Rook v1.13 supports the following Ceph versions:
+Rook v1.15 supports the following Ceph versions:
 
 * Ceph Reef v18.2.0 or newer
 * Ceph Quincy v17.2.0 or newer
 
-Support for Ceph Pacific (16.2.x) is removed in Rook v1.13. Upgrade to Quincy or Reef before upgrading
-to Rook v1.13.
-
 !!! important
     When an update is requested, the operator will check Ceph's status,
     **if it is in `HEALTH_ERR` the operator will refuse to proceed with the upgrade.**

Documentation/Upgrade/rook-upgrade.md

@@ -14,14 +14,15 @@ We welcome feedback and opening issues!
 
 ## Supported Versions
 
-This guide is for upgrading from **Rook v1.13.x to Rook v1.14.x**.
+This guide is for upgrading from **Rook v1.14.x to Rook v1.15.x**.
 
 Please refer to the upgrade guides from previous releases for supported upgrade paths.
 Rook upgrades are only supported between official releases.
 
 For a guide to upgrade previous versions of Rook, please refer to the version of documentation for
 those releases.
 
+* [Upgrade 1.13 to 1.14](https://rook.io/docs/rook/v1.14/Upgrade/rook-upgrade/)
 * [Upgrade 1.12 to 1.13](https://rook.io/docs/rook/v1.13/Upgrade/rook-upgrade/)
 * [Upgrade 1.11 to 1.12](https://rook.io/docs/rook/v1.12/Upgrade/rook-upgrade/)
 * [Upgrade 1.10 to 1.11](https://rook.io/docs/rook/v1.11/Upgrade/rook-upgrade/)
@@ -48,21 +49,21 @@ those releases.
     official releases. Builds from the master branch can have functionality changed or removed at any
     time without compatibility support and without prior notice.
 
-## Breaking changes in v1.14
+## Breaking changes in v1.15
 
-* The minimum supported version of Kubernetes is v1.25.
-    Upgrade to Kubernetes v1.25 or higher before upgrading Rook.
-* The Rook operator config `CSI_ENABLE_READ_AFFINITY` was removed. v1.13 clusters that have modified
-    this value to be `"true"` must set the option as desired in each CephCluster as documented
-    [here](https://rook.github.io/docs/rook/v1.14/CRDs/Cluster/ceph-cluster-crd/#csi-driver-options)
-    before upgrading to v1.14.
-* Rook is beginning the process of deprecating CSI network "holder" pods.
+* Rook has deprecated CSI network "holder" pods.
     If there are pods named `csi-*plugin-holder-*` in the Rook operator namespace, see the
     [detailed documentation](../CRDs/Cluster/network-providers.md#holder-pod-deprecation)
-    to disable them. This is optional for v1.14, but will be required in a future release.
-* In the operator helm chart, the images for the CSI driver are now specified with separate
-    `repository` and `tag` values. If the CSI images have been customized, convert them from the
-    `image` value to the separated `repository` and `tag` values.
+    to disable them. This deprecation process is required before upgrading to the future Rook v1.16.
+
+* Ceph COSI driver images have been updated. This impacts existing COSI Buckets, BucketClaims, and
+    BucketAccesses. Update existing clusters following the guide
+    [here](https://github.com/rook/rook/discussions/14297).
+
+* CephObjectStore, CephObjectStoreUser, and OBC endpoint behavior has changed when CephObjectStore
+    `spec.hosting` configurations are set. Use the new `spec.hosting.advertiseEndpoint` config to
+    define required behavior as
+    [documented](../Storage-Configuration/Object-Storage-RGW/object-storage.md#object-store-endpoint).
 
 
 ## Considerations
@@ -79,23 +80,23 @@ With this upgrade guide, there are a few notes to consider:
 
 Unless otherwise noted due to extenuating requirements, upgrades from one patch release of Rook to
 another are as simple as updating the common resources and the image of the Rook operator. For
-example, when Rook v1.14.1 is released, the process of updating from v1.14.0 is as simple as running
+example, when Rook v1.15.1 is released, the process of updating from v1.15.0 is as simple as running
 the following:
 
 ```console
-git clone --single-branch --depth=1 --branch v1.14.1 https://github.com/rook/rook.git
+git clone --single-branch --depth=1 --branch v1.15.1 https://github.com/rook/rook.git
 cd rook/deploy/examples
 ```
 
 If the Rook Operator or CephCluster are deployed into a different namespace than
 `rook-ceph`, see the [Update common resources and CRDs](#1-update-common-resources-and-crds)
 section for instructions on how to change the default namespaces in `common.yaml`.
 
-Then, apply the latest changes from v1.14, and update the Rook Operator image.
+Then, apply the latest changes from v1.15, and update the Rook Operator image.
 
 ```console
 kubectl apply -f common.yaml -f crds.yaml
-kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.14.1
+kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.15.1
 ```
 
 As exemplified above, it is a good practice to update Rook common resources from the example
@@ -112,7 +113,7 @@ The upgrade steps in this guide will clarify what Helm handles automatically.
 
 !!! important
     If there are pods named `csi-*plugin-holder-*` in the Rook operator namespace, set the new
-    config `csi.disableHolderPods: false` in the values.yaml before upgrading to v1.14.
+    config `csi.disableHolderPods: false` in the values.yaml before upgrading to v1.15.
 
 The `rook-ceph` helm chart upgrade performs the Rook upgrade.
 The `rook-ceph-cluster` helm chart upgrade performs a [Ceph upgrade](./ceph-upgrade.md) if the Ceph image is updated.
@@ -133,9 +134,9 @@ In order to successfully upgrade a Rook cluster, the following prerequisites mus
 
 ## Rook Operator Upgrade
 
-The examples given in this guide upgrade a live Rook cluster running `v1.13.7` to
-the version `v1.14.0`. This upgrade should work from any official patch release of Rook v1.13 to any
-official patch release of v1.14.
+The examples given in this guide upgrade a live Rook cluster running `v1.14.9` to
+the version `v1.15.0`. This upgrade should work from any official patch release of Rook v1.14 to any
+official patch release of v1.15.
 
 Let's get started!
 
@@ -196,7 +197,7 @@ kubectl apply -f deploy/examples/monitoring/rbac.yaml
 !!! hint
     The operator is automatically updated when using Helm charts.
 
-The largest portion of the upgrade is triggered when the operator's image is updated to `v1.14.x`.
+The largest portion of the upgrade is triggered when the operator's image is updated to `v1.15.x`.
 When the operator is updated, it will proceed to update all of the Ceph daemons.
 
 ```console
@@ -230,18 +231,18 @@ watch --exec kubectl -n $ROOK_CLUSTER_NAMESPACE get deployments -l rook_cluster=
 ```
 
 As an example, this cluster is midway through updating the OSDs. When all deployments report `1/1/1`
-availability and `rook-version=v1.14.0`, the Ceph cluster's core components are fully updated.
+availability and `rook-version=v1.15.0`, the Ceph cluster's core components are fully updated.
 
 ```console
 Every 2.0s: kubectl -n rook-ceph get deployment -o j...
 
-rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.14.0
-rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.14.0
-rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.14.0
-rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.14.0
-rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.14.0
-rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.13.7
-rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.13.7
+rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.15.0
+rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.15.0
+rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.15.0
+rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.15.0
+rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.15.0
+rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.14.9
+rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.14.9
 ```
 
 An easy check to see if the upgrade is totally finished is to check that there is only one
@@ -250,21 +251,21 @@ An easy check to see if the upgrade is totally finished is to check that there i
 ```console
 # kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"rook-version="}{.metadata.labels.rook-version}{"\n"}{end}' | sort | uniq
 This cluster is not yet finished:
-  rook-version=v1.13.7
-  rook-version=v1.14.0
+  rook-version=v1.14.9
+  rook-version=v1.15.0
 This cluster is finished:
-  rook-version=v1.14.0
+  rook-version=v1.15.0
 ```
 
 ### **5. Verify the updated cluster**
 
-At this point, the Rook operator should be running version `rook/ceph:v1.14.0`.
+At this point, the Rook operator should be running version `rook/ceph:v1.15.0`.
 
 Verify the CephCluster health using the [health verification doc](health-verification.md).
 
 ### **6. Disable holder pods**
 
-Rook is beginning the process of deprecating CSI network "holder" pods. If there are pods named
+Rook has deprecated CSI network "holder" pods. If there are pods named
 `csi-*plugin-holder-*` in the Rook operator namespace, see the
 [detailed documentation](../CRDs/Cluster/network-providers.md#holder-pod-deprecation)
-to disable them. This is optional for v1.14, but will be required in a future release.
+to disable them. This deprecation process is required before upgrading to the future Rook v1.16.

PendingReleaseNotes.md

@@ -2,15 +2,26 @@
 
 ## Breaking Changes
 
-- Updating Ceph COSI driver images, this impact existing COSI `Buckets` and `BucketAccesses`,
-please update the `BucketClass` and `BucketAccessClass` for resolving refer [here](https://github.com/rook/rook/discussions/14297)
-- During CephBlockPool updates, return an error if an invalid device class is specified. Pools with invalid device classes may start failing reconcile until the correct device class is specified. See #14057.
+- Rook has deprecated CSI network "holder" pods.
+    If there are pods named `csi-*plugin-holder-*` in the Rook operator namespace, see the
+    [detailed documentation](../CRDs/Cluster/network-providers.md#holder-pod-deprecation)
+    to disable them. This deprecation process is required before upgrading to the future Rook v1.16.
+- Ceph COSI driver images have been updated. This impacts existing COSI Buckets, BucketClaims, and
+    BucketAccesses. Update existing clusters following the guide
+    [here](https://github.com/rook/rook/discussions/14297).
+- During CephBlockPool updates, Rook will now return an error if an invalid device class is
+    specified. Pools with invalid device classes may start failing until the correct device class is
+    specified. For more info, see [#14057](https://github.com/rook/rook/pull/14057).
 - CephObjectStore, CephObjectStoreUser, and OBC endpoint behavior has changed when CephObjectStore
-  `spec.hosting` configurations are set. A new `spec.hosting.advertiseEndpoint` config was added to
-  allow users to define required behavior.
+    `spec.hosting` configurations are set. Use the new `spec.hosting.advertiseEndpoint` config to
+    define required behavior as
+    [documented](../Storage-Configuration/Object-Storage-RGW/object-storage.md#object-store-endpoint).
+- Minimum version of Kubernetes supported is increased to K8s v1.26.
 
 ## Features
 
 - Added support for Ceph Squid (v19)
 - Allow updating the device class of OSDs, if `allowDeviceClassUpdate: true` is set
-- Support for keystone authentication for s3 and swift (see [#9088](https://github.com/rook/rook/issues/9088)).
+- CephObjectStore support for keystone authentication for S3 and Swift
+    (see [#9088](https://github.com/rook/rook/issues/9088)).
+- Support K8s versions v1.26 through v1.31.