From facefbad6e32c9df646d2cbc606bb6885332bb22 Mon Sep 17 00:00:00 2001 From: Raunak Pradip Shah Date: Wed, 12 Jul 2023 11:37:13 +0530 Subject: [PATCH] add DeleteVolumeGroupSnapshot API to delete group snapshot --- cmd/snapshot-controller/main.go | 23 +- .../csi-snapshotter/rbac-csi-snapshotter.yaml | 12 +- .../rbac-snapshot-controller.yaml | 17 + .../groupsnapshot_controller_helper.go | 329 +++++++++++++++++- pkg/common-controller/snapshot_controller.go | 1 - .../snapshot_controller_base.go | 2 +- pkg/group_snapshotter/group_snapshotter.go | 21 +- pkg/sidecar-controller/csi_handler.go | 27 +- .../groupsnapshot_helper.go | 207 +++++++++-- pkg/utils/patch.go | 20 ++ pkg/utils/util.go | 40 +++ 11 files changed, 648 insertions(+), 51 deletions(-) diff --git a/cmd/snapshot-controller/main.go b/cmd/snapshot-controller/main.go index 3a8c53000..f62fab7cc 100644 --- a/cmd/snapshot-controller/main.go +++ b/cmd/snapshot-controller/main.go @@ -81,7 +81,7 @@ var ( var version = "unknown" // Checks that the VolumeSnapshot v1 CRDs exist. -func ensureCustomResourceDefinitionsExist(client *clientset.Clientset) error { +func ensureCustomResourceDefinitionsExist(client *clientset.Clientset, enableVolumeGroupSnapshots bool) error { condition := func() (bool, error) { var err error @@ -102,6 +102,25 @@ func ensureCustomResourceDefinitionsExist(client *clientset.Clientset) error { klog.Errorf("Failed to list v1 volumesnapshotcontents with error=%+v", err) return false, nil } + if enableVolumeGroupSnapshots { + _, err = client.GroupsnapshotV1alpha1().VolumeGroupSnapshots("").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to list v1alpha1 volumegroupsnapshots with error=%+v", err) + return false, nil + } + + _, err = client.GroupsnapshotV1alpha1().VolumeGroupSnapshotClasses().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to list v1alpha1 volumegroupsnapshotclasses with error=%+v", err) + return false, nil + } + _, err = client.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to list v1alpha1 volumegroupsnapshotcontents with error=%+v", err) + return false, nil + } + } + return true, nil } @@ -214,7 +233,7 @@ func main() { *enableVolumeGroupSnapshots, ) - if err := ensureCustomResourceDefinitionsExist(snapClient); err != nil { + if err := ensureCustomResourceDefinitionsExist(snapClient, *enableVolumeGroupSnapshots); err != nil { klog.Errorf("Exiting due to failure to ensure CRDs exist during startup: %+v", err) os.Exit(1) } diff --git a/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml b/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml index 6a33a5b17..6c15a63b0 100644 --- a/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml +++ b/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml @@ -36,11 +36,19 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "update", "patch", "create"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update", "patch"] - + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml b/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml index 04d01c3b3..eeadcfe01 100644 --- a/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml +++ b/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml @@ -42,6 +42,23 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] verbs: ["update", "patch"] + + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshots/status"] + verbs: ["update", "patch"] + # Enable this RBAC rule only when using distributed snapshotting, i.e. when the enable-distributed-snapshotting flag is set to true # - apiGroups: [""] # resources: ["nodes"] diff --git a/pkg/common-controller/groupsnapshot_controller_helper.go b/pkg/common-controller/groupsnapshot_controller_helper.go index bcb85f3f7..bc0b21c3c 100644 --- a/pkg/common-controller/groupsnapshot_controller_helper.go +++ b/pkg/common-controller/groupsnapshot_controller_helper.go @@ -289,13 +289,26 @@ func (ctrl *csiSnapshotCommonController) syncGroupSnapshot(groupSnapshot *crdv1a klog.V(5).Infof("syncGroupSnapshot [%s]: check if we should remove finalizer on group snapshot PVC source and remove it if we can", utils.GroupSnapshotKey(groupSnapshot)) - /* - TODO: - - Check and remove finalizer if needed. - - Check and set invalid group snapshot label, if needed. - - Process if deletion timestamp is set. - - Check and add group snapshot finalizers. - */ + // Proceed with group snapshot deletion and remove finalizers when needed + if groupSnapshot.ObjectMeta.DeletionTimestamp != nil { + return ctrl.processGroupSnapshotWithDeletionTimestamp(groupSnapshot) + } + // Keep this check in the controller since the validation webhook may not have been deployed. + klog.V(5).Infof("syncGroupSnapshot[%s]: validate group snapshot to make sure source has been correctly specified", utils.GroupSnapshotKey(groupSnapshot)) + if (&groupSnapshot.Spec.Source.Selector == nil && groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName == nil) || + (&groupSnapshot.Spec.Source.Selector != nil && groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName != nil) { + err := fmt.Errorf("Exactly one of Selector and VolumeGroupSnapshotContentName should be specified") + klog.Errorf("syncGroupSnapshot[%s]: validation error, %s", utils.GroupSnapshotKey(groupSnapshot), err.Error()) + ctrl.updateGroupSnapshotErrorStatusWithEvent(groupSnapshot, true, v1.EventTypeWarning, "GroupSnapshotValidationError", err.Error()) + return err + } + + klog.V(5).Infof("syncGroupSnapshot: check if we should add finalizers on group snapshot [%s]", utils.GroupSnapshotKey(groupSnapshot)) + if err := ctrl.checkandAddGroupSnapshotFinalizers(groupSnapshot); err != nil { + klog.Errorf("error checkandAddGroupSnapshotFinalizers for group snapshot [%s]: %v", utils.GroupSnapshotKey(groupSnapshot), err) + ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeWarning, "GroupSnapshotFinalizerError", fmt.Sprintf("Failed to check and update group snapshot: %s", err.Error())) + return err + } // Need to build or update groupSnapshot.Status in following cases: // 1) groupSnapshot.Status is nil @@ -724,10 +737,10 @@ func (ctrl *csiSnapshotCommonController) bindandUpdateVolumeGroupSnapshot(groupS // createGroupSnapshotContent will only be called for dynamic provisioning func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (*crdv1alpha1.VolumeGroupSnapshotContent, error) { - klog.Infof("createSnapshotContent: Creating group snapshot content for groupn snapshot %s through the plugin ...", utils.GroupSnapshotKey(groupSnapshot)) + klog.Infof("createSnapshotContent: Creating group snapshot content for group snapshot %s through the plugin ...", utils.GroupSnapshotKey(groupSnapshot)) /* - TODO: Add finalizer to group snapshot + TODO: Add PVC finalizer */ groupSnapshotClass, volumes, contentName, err := ctrl.getCreateGroupSnapshotInput(groupSnapshot) @@ -763,16 +776,16 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho TODO: Add secret reference details */ - var updateGroupSnapshoyContent *crdv1alpha1.VolumeGroupSnapshotContent + var updateGroupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent klog.V(5).Infof("volume group snapshot content %#v", groupSnapshotContent) // Try to create the VolumeGroupSnapshotContent object klog.V(5).Infof("createGroupSnapshotContent [%s]: trying to save volume group snapshot content %s", utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent.Name) - if updateGroupSnapshoyContent, err = ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Create(context.TODO(), groupSnapshotContent, metav1.CreateOptions{}); err == nil || apierrs.IsAlreadyExists(err) { + if updateGroupSnapshotContent, err = ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Create(context.TODO(), groupSnapshotContent, metav1.CreateOptions{}); err == nil || apierrs.IsAlreadyExists(err) { // Save succeeded. if err != nil { klog.V(3).Infof("volume group snapshot content %q for group snapshot %q already exists, reusing", groupSnapshotContent.Name, utils.GroupSnapshotKey(groupSnapshot)) err = nil - updateGroupSnapshoyContent = groupSnapshotContent + updateGroupSnapshotContent = groupSnapshotContent } else { klog.V(3).Infof("volume group snapshot content %q for group snapshot %q saved, %v", groupSnapshotContent.Name, utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent) } @@ -789,12 +802,12 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeNormal, "CreatingGroupSnapshot", msg) // Update group snapshot content in the cache store - _, err = ctrl.storeGroupSnapshotContentUpdate(updateGroupSnapshoyContent) + _, err = ctrl.storeGroupSnapshotContentUpdate(updateGroupSnapshotContent) if err != nil { klog.Errorf("failed to update group snapshot content store %v", err) } - return updateGroupSnapshoyContent, nil + return updateGroupSnapshotContent, nil } func (ctrl *csiSnapshotCommonController) getCreateGroupSnapshotInput(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (*crdv1alpha1.VolumeGroupSnapshotClass, []*v1.PersistentVolume, string, error) { @@ -849,9 +862,11 @@ func (ctrl *csiSnapshotCommonController) syncGroupSnapshotContent(groupSnapshotC return nil } - /* - TODO: Add finalizer to prevent deletion - */ + if utils.NeedToAddGroupSnapshotContentFinalizer(groupSnapshotContent) { + // Group Snapshot Content is not being deleted -> it should have the finalizer. + klog.V(5).Infof("syncGroupSnapshotContent [%s]: Add Finalizer for VolumeGroupSnapshotContent", groupSnapshotContent.Name) + return ctrl.addGroupSnapshotContentFinalizer(groupSnapshotContent) + } // Check if group snapshot exists in cache store // If getGroupSnapshotFromStore returns (nil, nil), it means group snapshot not found @@ -935,3 +950,283 @@ func (ctrl *csiSnapshotCommonController) needsUpdateGroupSnapshotStatus(groupSna return false } + +// addGroupSnapshotContentFinalizer adds a Finalizer for VolumeGroupSnapshotContent. +func (ctrl *csiSnapshotCommonController) addGroupSnapshotContentFinalizer(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error { + var patches []utils.PatchOp + if len(groupSnapshotContent.Finalizers) > 0 { + // Add to the end of the finalizers if we have any other finalizers + patches = append(patches, utils.PatchOp{ + Op: "add", + Path: "/metadata/finalizers/-", + Value: utils.VolumeGroupSnapshotContentFinalizer, + }) + } else { + // Replace finalizers with new array if there are no other finalizers + patches = append(patches, utils.PatchOp{ + Op: "add", + Path: "/metadata/finalizers", + Value: []string{utils.VolumeGroupSnapshotContentFinalizer}, + }) + } + newGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContent, patches, ctrl.clientset) + if err != nil { + return newControllerUpdateError(groupSnapshotContent.Name, err.Error()) + } + + _, err = ctrl.storeGroupSnapshotContentUpdate(newGroupSnapshotContent) + if err != nil { + klog.Errorf("failed to update group snapshot content store %v", err) + } + + klog.V(5).Infof("Added protection finalizer to volume group snapshot content %s", newGroupSnapshotContent.Name) + return nil +} + +// checkandAddGroupSnapshotFinalizers checks and adds group snapshot finailzers when needed +func (ctrl *csiSnapshotCommonController) checkandAddGroupSnapshotFinalizers(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) error { + // get the group snapshot content for this group snapshot + var ( + groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent + err error + ) + if groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName != nil { + groupSnapshotContent, err = ctrl.getPreprovisionedGroupSnapshotContentFromStore(groupSnapshot) + } else { + groupSnapshotContent, err = ctrl.getDynamicallyProvisionedGroupContentFromStore(groupSnapshot) + } + if err != nil { + return err + } + + // A bound finalizer is needed ONLY when all following conditions are satisfied: + // 1. the VolumeGroupSnapshot is bound to a VolumeGroupSnapshotContent + // 2. the VolumeGroupSnapshot does not have deletion timestamp set + // 3. the matching VolumeGroupSnapshotContent has a deletion policy to be Delete + // Note that if a matching VolumeGroupSnapshotContent is found, it must point back to the VolumeGroupSnapshot + if groupSnapshotContent != nil && utils.NeedToAddGroupSnapshotBoundFinalizer(groupSnapshot) && (groupSnapshotContent.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete) { + // Snapshot is not being deleted -> it should have the finalizer. + klog.V(5).Infof("checkandAddGroupSnapshotFinalizers: Add Finalizer for VolumeGroupSnapshot[%s]", utils.GroupSnapshotKey(groupSnapshot)) + return ctrl.addGroupSnapshotFinalizer(groupSnapshot, true) + + } + return nil +} + +// addGroupSnapshotFinalizer adds a Finalizer to a VolumeGroupSnapshot. +func (ctrl *csiSnapshotCommonController) addGroupSnapshotFinalizer(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot, addBoundFinalizer bool) error { + var updatedGroupSnapshot *crdv1alpha1.VolumeGroupSnapshot + var err error + + // NOTE(ggriffiths): Must perform an update if no finalizers exist. + // Unable to find a patch that correctly updated the finalizers if none currently exist. + if len(groupSnapshot.ObjectMeta.Finalizers) == 0 { + groupSnapshotClone := groupSnapshot.DeepCopy() + if addBoundFinalizer { + groupSnapshotClone.ObjectMeta.Finalizers = append(groupSnapshotClone.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotBoundFinalizer) + } + updatedGroupSnapshot, err = ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).Update(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{}) + if err != nil { + return newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), err.Error()) + } + } else { + // Otherwise, perform a patch + var patches []utils.PatchOp + + if addBoundFinalizer { + patches = append(patches, utils.PatchOp{ + Op: "add", + Path: "/metadata/finalizers/-", + Value: utils.VolumeGroupSnapshotBoundFinalizer, + }) + } + + updatedGroupSnapshot, err = utils.PatchVolumeGroupSnapshot(groupSnapshot, patches, ctrl.clientset) + if err != nil { + return newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), err.Error()) + } + } + + _, err = ctrl.storeGroupSnapshotUpdate(updatedGroupSnapshot) + if err != nil { + klog.Errorf("failed to update group snapshot store %v", err) + } + + klog.V(5).Infof("Added protection finalizer to volume group snapshot %s", utils.GroupSnapshotKey(updatedGroupSnapshot)) + return nil +} + +// processGroupSnapshotWithDeletionTimestamp processes finalizers and deletes the +// group snapshot content when appropriate. It has the following steps: +// 1. Get the VolumeGroupSnapshotContent which the to-be-deleted VolumeGroupSnapshot +// points to and verifies bi-directional binding. +// 2. Call checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent() +// with information obtained from step 1. This function name is very long but the +// name suggests what it does. It determines whether to remove finalizers on group +// snapshot and whether to delete group snapshot content. +func (ctrl *csiSnapshotCommonController) processGroupSnapshotWithDeletionTimestamp(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) error { + klog.V(5).Infof("processGroupSnapshotWithDeletionTimestamp VolumeGroupSnapshot[%s]: %s", utils.GroupSnapshotKey(groupSnapshot), utils.GetGroupSnapshotStatusForLogging(groupSnapshot)) + + var groupSnapshotContentName string + if groupSnapshot.Status != nil && groupSnapshot.Status.BoundVolumeGroupSnapshotContentName != nil { + groupSnapshotContentName = *groupSnapshot.Status.BoundVolumeGroupSnapshotContentName + } + // for a dynamically created group snapshot, it's possible that a group snapshot + // content has been created however the Status of the group snapshot has not + // been updated yet, i.e., failed right after group snapshot content creation. + // In this case, use the fixed naming scheme to get the group snapshot content + // name and search + if groupSnapshotContentName == "" && &groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName == nil { + groupSnapshotContentName = utils.GetDynamicSnapshotContentNameForGroupSnapshot(groupSnapshot) + } + // find a group snapshot content from cache store, note that it's completely legit + // that no group snapshot content has been found from group snapshot content + // cache store + groupSnapshotContent, err := ctrl.getGroupSnapshotContentFromStore(groupSnapshotContentName) + if err != nil { + return err + } + // check whether the group snapshot content points back to the passed in group + // snapshot, note that binding should always be bi-directional to trigger the + // deletion on group snapshot content or adding any annotation to the group + // snapshot content + var deleteGroupSnapshotContent bool + if groupSnapshotContent != nil && utils.IsVolumeGroupSnapshotRefSet(groupSnapshot, groupSnapshotContent) { + // group snapshot content points back to group snapshot, whether or not + // to delete a group snapshot content now depends on the deletion policy + // of it. + deleteGroupSnapshotContent = (groupSnapshotContent.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete) + } else { + // the group snapshot content is nil or points to a different group snapshot, reset group snapshot content to nil + // such that there is no operation done on the found group snapshot content in + // checkandRemoveSnapshotFinalizersAndCheckandDeleteContent + groupSnapshotContent = nil + } + + klog.V(5).Infof("processGroupSnapshotWithDeletionTimestamp[%s]: delete group snapshot content and remove finalizer from group snapshot if needed", utils.GroupSnapshotKey(groupSnapshot)) + + return ctrl.checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent(groupSnapshot, groupSnapshotContent, deleteGroupSnapshotContent) +} + +// checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent deletes +// the group snapshot content and removes group snapshot finalizers if needed +func (ctrl *csiSnapshotCommonController) checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot, groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent, deleteGroupSnapshotContent bool) error { + klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent VolumeGroupSnapshot[%s]: %s", utils.GroupSnapshotKey(groupSnapshot), utils.GetGroupSnapshotStatusForLogging(groupSnapshot)) + + if !utils.IsGroupSnapshotDeletionCandidate(groupSnapshot) { + return nil + } + + // check if an individual snapshot belonging to the group snapshot is being + // used for restore a PVC + // If yes, do nothing and wait until PVC restoration finishes + for _, snapshotRef := range groupSnapshot.Status.VolumeSnapshotRefList { + snapshot, err := ctrl.snapshotLister.VolumeSnapshots(snapshotRef.Namespace).Get(snapshotRef.Name) + if err != nil { + return err + } + if ctrl.isVolumeBeingCreatedFromSnapshot(snapshot) { + msg := fmt.Sprintf("Snapshot %s belonging to VolumeGroupSnapshot %s is being used to restore a PVC", utils.SnapshotKey(snapshot), utils.GroupSnapshotKey(groupSnapshot)) + klog.V(4).Info(msg) + ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeWarning, "SnapshotDeletePending", msg) + // TODO(@xiangqian): should requeue this? + return nil + } + + } + + // regardless of the deletion policy, set the VolumeSnapshotBeingDeleted on + // content object, this is to allow snapshotter sidecar controller to conduct + // a delete operation whenever the content has deletion timestamp set. + if groupSnapshotContent != nil { + klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent[%s]: Set VolumeGroupSnapshotBeingDeleted annotation on the group snapshot content [%s]", utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent.Name) + updatedGroupSnapshotContent, err := ctrl.setAnnVolumeGroupSnapshotBeingDeleted(groupSnapshotContent) + if err != nil { + klog.V(4).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent[%s]: failed to set VolumeGroupSnapshotBeingDeleted annotation on the group snapshot content [%s]", utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent.Name) + return err + } + groupSnapshotContent = updatedGroupSnapshotContent + } + + // VolumeGroupSnapshot should be deleted. Check and remove finalizers + // If group snapshot content exists and has a deletion policy of Delete, set + // DeletionTimeStamp on the content; + // VolumeGroupSnapshotContent won't be deleted immediately due to the VolumeGroupSnapshotContentFinalizer + if groupSnapshotContent != nil && deleteGroupSnapshotContent { + klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent: set DeletionTimeStamp on group snapshot content [%s].", groupSnapshotContent.Name) + err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Delete(context.TODO(), groupSnapshotContent.Name, metav1.DeleteOptions{}) + if err != nil { + ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeWarning, "GroupSnapshotContentObjectDeleteError", "Failed to delete group snapshot content API object") + return fmt.Errorf("failed to delete VolumeGroupSnapshotContent %s from API server: %q", groupSnapshotContent.Name, err) + } + } + + klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent: Remove Finalizer for VolumeGroupSnapshot[%s]", utils.GroupSnapshotKey(groupSnapshot)) + // remove VolumeSnapshotBoundFinalizer on the VolumeGroupSnapshot object: + // a. If there is no group snapshot content found, remove the finalizer. + // b. If the group snapshot content is being deleted, i.e., with deleteGroupSnapshotContent == true, + // keep this finalizer until the group snapshot content object is removed + // from API server by group snapshot sidecar controller. + // c. If deletion will not cascade to the group snapshot content, remove + // the finalizer on the group snapshot such that it can be removed from + // the API server. + removeBoundFinalizer := !(groupSnapshotContent != nil && deleteGroupSnapshotContent) + return ctrl.removeGroupSnapshotFinalizer(groupSnapshot, removeBoundFinalizer) +} + +func (ctrl *csiSnapshotCommonController) setAnnVolumeGroupSnapshotBeingDeleted(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) (*crdv1alpha1.VolumeGroupSnapshotContent, error) { + if groupSnapshotContent == nil { + return groupSnapshotContent, nil + } + // Set AnnVolumeGroupSnapshotBeingDeleted if it is not set yet + if !metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingDeleted) { + klog.V(5).Infof("setAnnVolumeGroupSnapshotBeingDeleted: set annotation [%s] on group snapshot content [%s].", utils.AnnVolumeGroupSnapshotBeingDeleted, groupSnapshotContent.Name) + var patches []utils.PatchOp + metav1.SetMetaDataAnnotation(&groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingDeleted, "yes") + patches = append(patches, utils.PatchOp{ + Op: "replace", + Path: "/metadata/annotations", + Value: groupSnapshotContent.ObjectMeta.GetAnnotations(), + }) + + patchedGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContent, patches, ctrl.clientset) + if err != nil { + return groupSnapshotContent, newControllerUpdateError(groupSnapshotContent.Name, err.Error()) + } + + // update group snapshot content if update is successful + groupSnapshotContent = patchedGroupSnapshotContent + + _, err = ctrl.storeGroupSnapshotContentUpdate(groupSnapshotContent) + if err != nil { + klog.V(4).Infof("setAnnVolumeGroupSnapshotBeingDeleted for group snapshot content [%s]: cannot update internal cache %v", groupSnapshotContent.Name, err) + return groupSnapshotContent, err + } + klog.V(5).Infof("setAnnVolumeGroupSnapshotBeingDeleted: volume group snapshot content %+v", groupSnapshotContent) + } + return groupSnapshotContent, nil +} + +// removeGroupSnapshotFinalizer removes a Finalizer for VolumeGroupSnapshot. +func (ctrl *csiSnapshotCommonController) removeGroupSnapshotFinalizer(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot, removeBoundFinalizer bool) error { + if !removeBoundFinalizer { + return nil + } + + // TODO: Remove PVC Finalizer + + groupSnapshotClone := groupSnapshot.DeepCopy() + groupSnapshotClone.ObjectMeta.Finalizers = utils.RemoveString(groupSnapshotClone.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotBoundFinalizer) + newGroupSnapshot, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).Update(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{}) + if err != nil { + return newControllerUpdateError(groupSnapshot.Name, err.Error()) + } + + _, err = ctrl.storeGroupSnapshotUpdate(newGroupSnapshot) + if err != nil { + klog.Errorf("failed to update group snapshot store %v", err) + } + + klog.V(5).Infof("Removed protection finalizer from volume group snapshot %s", utils.GroupSnapshotKey(groupSnapshot)) + return nil +} diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index 8ae79dfae..2b1d051e9 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -1165,7 +1165,6 @@ func (ctrl *csiSnapshotCommonController) updateSnapshotStatus(snapshot *crdv1.Vo updated = true } else { newStatus = snapshotObj.Status.DeepCopy() - klog.Infof("Raunak 1 %s", newStatus.VolumeGroupSnapshotName) if newStatus.BoundVolumeSnapshotContentName == nil { newStatus.BoundVolumeSnapshotContentName = &boundContentName updated = true diff --git a/pkg/common-controller/snapshot_controller_base.go b/pkg/common-controller/snapshot_controller_base.go index 1f9ecad9c..ad9ce0432 100644 --- a/pkg/common-controller/snapshot_controller_base.go +++ b/pkg/common-controller/snapshot_controller_base.go @@ -836,7 +836,7 @@ func (ctrl *csiSnapshotCommonController) updateGroupSnapshotContent(content *crd // deleteGroupSnapshotContent runs in worker thread and handles "groupsnapshotcontent deleted" event. func (ctrl *csiSnapshotCommonController) deleteGroupSnapshotContent(content *crdv1alpha1.VolumeGroupSnapshotContent) { - _ = ctrl.contentStore.Delete(content) + _ = ctrl.groupSnapshotContentStore.Delete(content) klog.V(4).Infof("group snapshot content %q deleted", content.Name) groupSnapshotName := utils.GroupSnapshotRefKey(&content.Spec.VolumeGroupSnapshotRef) diff --git a/pkg/group_snapshotter/group_snapshotter.go b/pkg/group_snapshotter/group_snapshotter.go index 8cfe6103f..8a08507a0 100644 --- a/pkg/group_snapshotter/group_snapshotter.go +++ b/pkg/group_snapshotter/group_snapshotter.go @@ -18,11 +18,12 @@ package group_snapshotter import ( "context" + "time" + "github.com/container-storage-interface/spec/lib/go/csi" csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" "google.golang.org/grpc" klog "k8s.io/klog/v2" - "time" ) // GroupSnapshotter implements CreateGroupSnapshot/DeleteGroupSnapshot operations against a CSI driver. @@ -31,7 +32,7 @@ type GroupSnapshotter interface { CreateGroupSnapshot(ctx context.Context, groupSnapshotName string, volumeIDs []string, parameters map[string]string, snapshotterCredentials map[string]string) (driverName string, groupSnapshotId string, snapshots []*csi.Snapshot, timestamp time.Time, readyToUse bool, err error) // DeleteGroupSnapshot deletes a group snapshot of multiple volumes - DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotterCredentials map[string]string) (err error) + DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotIDs []string, snapshotterCredentials map[string]string) (err error) // GetGroupSnapshotStatus returns if a group snapshot is ready to use, its creation time, etc GetGroupSnapshotStatus(ctx context.Context, groupSnapshotID string, snapshotterListCredentials map[string]string) (bool, time.Time, error) @@ -74,8 +75,20 @@ func (gs *groupSnapshot) CreateGroupSnapshot(ctx context.Context, groupSnapshotN } -func (gs *groupSnapshot) DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotterCredentials map[string]string) error { - // TODO: Implement DeleteGroupSnapshot +func (gs *groupSnapshot) DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotIds []string, snapshotterCredentials map[string]string) error { + client := csi.NewGroupControllerClient(gs.conn) + + req := csi.DeleteVolumeGroupSnapshotRequest{ + Secrets: snapshotterCredentials, + GroupSnapshotId: groupSnapshotID, + SnapshotIds: snapshotIds, + } + + _, err := client.DeleteVolumeGroupSnapshot(ctx, &req) + if err != nil { + return err + } + return nil } diff --git a/pkg/sidecar-controller/csi_handler.go b/pkg/sidecar-controller/csi_handler.go index 159900706..711e7bac3 100644 --- a/pkg/sidecar-controller/csi_handler.go +++ b/pkg/sidecar-controller/csi_handler.go @@ -19,11 +19,12 @@ package sidecar_controller import ( "context" "fmt" + "strings" + "time" + "github.com/container-storage-interface/spec/lib/go/csi" crdv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumegroupsnapshot/v1alpha1" "github.com/kubernetes-csi/external-snapshotter/v6/pkg/group_snapshotter" - "strings" - "time" crdv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/kubernetes-csi/external-snapshotter/v6/pkg/snapshotter" @@ -36,6 +37,7 @@ type Handler interface { GetSnapshotStatus(content *crdv1.VolumeSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, int64, error) CreateGroupSnapshot(content *crdv1alpha1.VolumeGroupSnapshotContent, volumeIDs []string, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, []*csi.Snapshot, time.Time, bool, error) GetGroupSnapshotStatus(content *crdv1alpha1.VolumeGroupSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, error) + DeleteGroupSnapshot(content *crdv1alpha1.VolumeGroupSnapshotContent, SnapshotID []string, snapshotterCredentials map[string]string) error } // csiHandler is a handler that calls CSI to create/delete volume snapshot. @@ -165,6 +167,27 @@ func (handler *csiHandler) CreateGroupSnapshot(content *crdv1alpha1.VolumeGroupS return handler.groupSnapshotter.CreateGroupSnapshot(ctx, groupSnapshotName, volumeIDs, parameters, snapshotterCredentials) } +func (handler *csiHandler) DeleteGroupSnapshot(content *crdv1alpha1.VolumeGroupSnapshotContent, snapshotIDs []string, snapshotterCredentials map[string]string) error { + ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) + defer cancel() + + // NOTE: snapshotIDs are required for DeleteGroupSnapshot + if len(snapshotIDs) == 0 { + return fmt.Errorf("cannot delete group snapshot content %s. No snapshots found in the group snapshot", content.Name) + } + var groupSnapshotHandle string + + if content.Status != nil && content.Status.VolumeGroupSnapshotHandle != nil { + groupSnapshotHandle = *content.Status.VolumeGroupSnapshotHandle + } else if content.Spec.Source.VolumeGroupSnapshotHandle != nil { + groupSnapshotHandle = *content.Spec.Source.VolumeGroupSnapshotHandle + } else { + return fmt.Errorf("failed to delete group snapshot content %s: groupsnapshotHandle is missing", content.Name) + } + + return handler.groupSnapshotter.DeleteGroupSnapshot(ctx, groupSnapshotHandle, snapshotIDs, snapshotterCredentials) +} + func (handler *csiHandler) GetGroupSnapshotStatus(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, error) { ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() diff --git a/pkg/sidecar-controller/groupsnapshot_helper.go b/pkg/sidecar-controller/groupsnapshot_helper.go index c8773993e..f438425a0 100644 --- a/pkg/sidecar-controller/groupsnapshot_helper.go +++ b/pkg/sidecar-controller/groupsnapshot_helper.go @@ -20,6 +20,7 @@ import ( "context" "crypto/sha256" "fmt" + "strings" "time" v1 "k8s.io/api/core/v1" @@ -163,9 +164,23 @@ func (ctrl *csiSnapshotSideCarController) deleteGroupSnapshotContentInCacheStore func (ctrl *csiSnapshotSideCarController) syncGroupSnapshotContent(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error { klog.V(5).Infof("synchronizing VolumeGroupSnapshotContent[%s]", groupSnapshotContent.Name) - /* - TODO: Check if the group snapshot content should be deleted - */ + if ctrl.shouldDeleteGroupSnapshotContent(groupSnapshotContent) { + klog.V(4).Infof("VolumeGroupSnapshotContent[%s]: the policy is %s", groupSnapshotContent.Name, groupSnapshotContent.Spec.DeletionPolicy) + if groupSnapshotContent.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete && + groupSnapshotContent.Status != nil && groupSnapshotContent.Status.VolumeGroupSnapshotHandle != nil { + // issue a CSI deletion call if the group snapshot has not been deleted + // yet from underlying storage system. Note that the delete group snapshot + // operation will update groups snapshot content's GroupSnapshotHandle + // to nil upon a successful deletion. At this point, the finalizer on + // group snapshot content should NOT be removed to avoid leaking. + return ctrl.deleteCSIGroupSnapshotOperation(groupSnapshotContent) + } + // otherwise, either the snapshot has been deleted from the underlying + // storage system, or the deletion policy is Retain, remove the finalizer + // if there is one so that API server could delete the object if there is + // no other finalizer. + return ctrl.removeGroupSnapshotContentFinalizer(groupSnapshotContent) + } if len(groupSnapshotContent.Spec.Source.PersistentVolumeNames) != 0 && groupSnapshotContent.Status == nil { klog.V(5).Infof("syncGroupSnapshotContent: Call CreateGroupSnapshot for group snapshot content %s", groupSnapshotContent.Name) @@ -184,6 +199,158 @@ func (ctrl *csiSnapshotSideCarController) syncGroupSnapshotContent(groupSnapshot return ctrl.checkandUpdateGroupSnapshotContentStatus(groupSnapshotContent) } +// removeGroupSnapshotContentFinalizer removes the VolumeGroupSnapshotContentFinalizer from a +// group snapshot content if there exists one. +func (ctrl csiSnapshotSideCarController) removeGroupSnapshotContentFinalizer(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error { + if !utils.ContainsString(groupSnapshotContent.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotContentFinalizer) { + // the finalizer does not exit, return directly + return nil + } + var patches []utils.PatchOp + groupSnapshotContentClone := groupSnapshotContent.DeepCopy() + patches = append(patches, + utils.PatchOp{ + Op: "replace", + Path: "/metadata/finalizers", + Value: utils.RemoveString(groupSnapshotContentClone.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotContentFinalizer), + }) + + updatedGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContentClone, patches, ctrl.clientset) + if err != nil { + return newControllerUpdateError(groupSnapshotContent.Name, err.Error()) + } + + klog.V(5).Infof("Removed protection finalizer from volume group snapshot content %s", updatedGroupSnapshotContent.Name) + _, err = ctrl.storeGroupSnapshotContentUpdate(updatedGroupSnapshotContent) + if err != nil { + klog.Errorf("failed to update group snapshot content store %v", err) + } + return nil +} + +// Delete a groupsnapshot: Ask the backend to remove the groupsnapshot device +func (ctrl *csiSnapshotSideCarController) deleteCSIGroupSnapshotOperation(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error { + klog.V(5).Infof("deleteCSISnapshotOperation [%s] started", groupSnapshotContent.Name) + + snapshotterCredentials, err := ctrl.GetCredentialsFromAnnotationForGroupSnapshot(groupSnapshotContent) + if err != nil { + ctrl.eventRecorder.Event(groupSnapshotContent, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to get snapshot credentials") + return fmt.Errorf("failed to get input parameters to delete group snapshot for group snapshot content %s: %q", groupSnapshotContent.Name, err) + } + + var snapshotIDs []string + if groupSnapshotContent.Status != nil && len(groupSnapshotContent.Status.VolumeSnapshotContentRefList) != 0 { + for _, contentRef := range groupSnapshotContent.Status.VolumeSnapshotContentRefList { + snapshotContent, err := ctrl.contentLister.Get(contentRef.Name) + if err != nil { + return fmt.Errorf("failed to get snapshot content %s from snapshot content store: %v", contentRef.Name, err) + } + snapshotIDs = append(snapshotIDs, *snapshotContent.Status.SnapshotHandle) + } + } + + err = ctrl.handler.DeleteGroupSnapshot(groupSnapshotContent, snapshotIDs, snapshotterCredentials) + if err != nil { + ctrl.eventRecorder.Event(groupSnapshotContent, v1.EventTypeWarning, "GroupSnapshotDeleteError", "Failed to delete group snapshot") + return fmt.Errorf("failed to delete group snapshot %#v, err: %v", groupSnapshotContent.Name, err) + } + // the group snapshot has been deleted from the underlying storage system, update + // group snapshot content status to remove the group snapshot handle etc. + newContent, err := ctrl.clearGroupSnapshotContentStatus(groupSnapshotContent.Name) + if err != nil { + ctrl.eventRecorder.Event(groupSnapshotContent, v1.EventTypeWarning, "GroupSnapshotDeleteError", "Failed to clear content status") + return err + } + // trigger syncGroupSnapshotContent + ctrl.updateGroupSnapshotContentInInformerCache(newContent) + return nil +} + +// clearGroupSnapshotContentStatus resets all fields to nil related to a group snapshot +// in groupSnapshotContent.Status. On success, the latest version of the group snapshot +// content object will be returned. +func (ctrl *csiSnapshotSideCarController) clearGroupSnapshotContentStatus( + groupSnapshotContentName string) (*crdv1alpha1.VolumeGroupSnapshotContent, error) { + klog.V(5).Infof("clearGroupSnapshotContentStatus content [%s]", groupSnapshotContentName) + // get the latest version from API server + groupSnapshotContent, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Get(context.TODO(), groupSnapshotContentName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error get group snapshot content %s from api server: %v", groupSnapshotContentName, err) + } + if groupSnapshotContent.Status != nil { + groupSnapshotContent.Status.VolumeGroupSnapshotHandle = nil + groupSnapshotContent.Status.ReadyToUse = nil + groupSnapshotContent.Status.CreationTime = nil + groupSnapshotContent.Status.Error = nil + groupSnapshotContent.Status.VolumeSnapshotContentRefList = nil + } + newContent, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().UpdateStatus(context.TODO(), groupSnapshotContent, metav1.UpdateOptions{}) + if err != nil { + return groupSnapshotContent, newControllerUpdateError(groupSnapshotContentName, err.Error()) + } + return newContent, nil +} + +func (ctrl *csiSnapshotSideCarController) GetCredentialsFromAnnotationForGroupSnapshot(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) (map[string]string, error) { + // get secrets if VolumeGroupSnapshotClass specifies it + var snapshotterCredentials map[string]string + var err error + + // Check if annotation exists + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefName) && metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefNamespace) { + annDeletionSecretName := groupSnapshotContent.Annotations[utils.AnnDeletionSecretRefName] + annDeletionSecretNamespace := groupSnapshotContent.Annotations[utils.AnnDeletionSecretRefNamespace] + + snapshotterSecretRef := &v1.SecretReference{} + + if annDeletionSecretName == "" || annDeletionSecretNamespace == "" { + return nil, fmt.Errorf("cannot retrieve secrets for group snapshot content %#v, err: secret name or namespace not specified", groupSnapshotContent.Name) + } + + snapshotterSecretRef.Name = annDeletionSecretName + snapshotterSecretRef.Namespace = annDeletionSecretNamespace + + snapshotterCredentials, err = utils.GetCredentials(ctrl.client, snapshotterSecretRef) + if err != nil { + // Continue with deletion, as the secret may have already been deleted. + klog.Errorf("Failed to get credentials for group snapshot content %s: %s", groupSnapshotContent.Name, err.Error()) + return nil, fmt.Errorf("cannot get credentials for group snapshot content %#v", groupSnapshotContent.Name) + } + } + + return snapshotterCredentials, nil +} + +// shouldDeleteGroupSnapshotContent checks if groupSnapshotContent object should be deleted +// if DeletionTimestamp is set on the groupSnapshotContent +func (ctrl *csiSnapshotSideCarController) shouldDeleteGroupSnapshotContent(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) bool { + klog.V(5).Infof("Check if VolumeGroupSnapshotContent[%s] should be deleted.", groupSnapshotContent.Name) + + if groupSnapshotContent.ObjectMeta.DeletionTimestamp == nil { + return false + } + // 1) shouldDeleteGroupSnapshot returns true if a content is not bound + // (VolumeGroupSnapshotRef == "") for pre-provisioned snapshot + if groupSnapshotContent.Spec.Source.VolumeGroupSnapshotHandle != nil && groupSnapshotContent.Spec.VolumeGroupSnapshotRef.UID == "" { + return true + } + + // NOTE(xyang): Handle create snapshot timeout + // 2) shouldDeleteGroupSnapshotContent returns false if AnnVolumeGroupSnapshotBeingCreated + // annotation is set. This indicates a CreateGroupSnapshot CSI RPC has + // not responded with success or failure. + // We need to keep waiting for a response from the CSI driver. + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingCreated) { + return false + } + + // 3) shouldDeleteGroupSnapshotContent returns true if AnnVolumeSnapshotBeingDeleted annotation is set + if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingDeleted) { + return true + } + return false +} + // createGroupSnapshot starts new asynchronous operation to create group snapshot func (ctrl *csiSnapshotSideCarController) createGroupSnapshot(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error { klog.V(5).Infof("createGroupSnapshot for group snapshot content [%s]: started", groupSnapshotContent.Name) @@ -289,9 +456,9 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh VolumeGroupSnapshotContentName: &groupSnapshotContent.Name, }, } + label := make(map[string]string) label["volumeGroupSnapshotName"] = groupSnapshotContent.Spec.VolumeGroupSnapshotRef.Name - name := "f" volumeSnapshot := &crdv1.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: volumeSnapshotName, @@ -304,29 +471,19 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh }, }, } + vsc, err := ctrl.clientset.SnapshotV1().VolumeSnapshotContents().Create(context.TODO(), volumeSnapshotContent, metav1.CreateOptions{}) if err != nil { return groupSnapshotContent, err } snapshotContentNames = append(snapshotContentNames, vsc.Name) - klog.Infof("making snapshot %v %s %s", volumeSnapshot.Status, *volumeSnapshot.Status.VolumeGroupSnapshotName, name) _, err = ctrl.clientset.SnapshotV1().VolumeSnapshots(volumeSnapshotNamespace).Create(context.TODO(), volumeSnapshot, metav1.CreateOptions{}) if err != nil { return groupSnapshotContent, err } - // klog.Infof("raunak made snapshot 1 %v", spew.Sdump(sn)) - // sn.Status = &crdv1.VolumeSnapshotStatus{ - // VolumeGroupSnapshotName: &name, - // } - // sn, err = ctrl.clientset.SnapshotV1().VolumeSnapshots(volumeSnapshotNamespace).UpdateStatus(context.TODO(), sn, metav1.UpdateOptions{}) - // if err != nil { - // klog.Infof("failed 2") - // return groupSnapshotContent, err - // } - // klog.Infof("made snapshot 2 %v", spew.Sdump(sn)) - } - klog.Infof("raunak 2") + } + newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotContentNames) if err != nil { klog.Errorf("error updating status for volume group snapshot content %s: %v.", groupSnapshotContent.Name, err) @@ -452,19 +609,25 @@ func (ctrl csiSnapshotSideCarController) removeAnnVolumeGroupSnapshotBeingCreate return groupSnapshotContent, nil } groupSnapshotContentClone := groupSnapshotContent.DeepCopy() - delete(groupSnapshotContentClone.ObjectMeta.Annotations, utils.AnnVolumeGroupSnapshotBeingCreated) + annotationPatchPath := strings.ReplaceAll(utils.AnnVolumeGroupSnapshotBeingCreated, "/", "~1") + + var patches []utils.PatchOp + patches = append(patches, utils.PatchOp{ + Op: "remove", + Path: "/metadata/annotations/" + annotationPatchPath, + }) - updatedContent, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Update(context.TODO(), groupSnapshotContentClone, metav1.UpdateOptions{}) + updatedGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContentClone, patches, ctrl.clientset) if err != nil { return groupSnapshotContent, newControllerUpdateError(groupSnapshotContent.Name, err.Error()) } klog.V(5).Infof("Removed VolumeGroupSnapshotBeingCreated annotation from volume group snapshot content %s", groupSnapshotContent.Name) - _, err = ctrl.storeContentUpdate(updatedContent) + _, err = ctrl.storeGroupSnapshotContentUpdate(updatedGroupSnapshotContent) if err != nil { klog.Errorf("failed to update groupSnapshotContent store %v", err) } - return updatedContent, nil + return updatedGroupSnapshotContent, nil } func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus( @@ -671,7 +834,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateGroupSnapshotContentStat driverName = groupSnapshotContent.Spec.Driver groupSnapshotID = *groupSnapshotContent.Spec.Source.VolumeGroupSnapshotHandle - klog.V(5).Infof("checkandUpdateGroupSnapshotContentStatusOperation: driver %s, groupSnapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, groupSnapshotID, creationTime, readyToUse) + klog.V(5).Infof("checkandUpdateGroupSnapshotContentStatusOperation: driver %s, groupSnapshotId %s, creationTime %v, readyToUse %t", driverName, groupSnapshotID, creationTime, readyToUse) if creationTime.IsZero() { creationTime = time.Now() diff --git a/pkg/utils/patch.go b/pkg/utils/patch.go index 724ca1a66..4443ef08b 100644 --- a/pkg/utils/patch.go +++ b/pkg/utils/patch.go @@ -58,6 +58,26 @@ func PatchVolumeSnapshot( return newSnapshot, nil } +// PatchVolumeGroupSnapshot patches a volume group snapshot object +func PatchVolumeGroupSnapshot( + existingGroupSnapshot *crdv1alpha1.VolumeGroupSnapshot, + patch []PatchOp, + client clientset.Interface, + subresources ...string, +) (*crdv1alpha1.VolumeGroupSnapshot, error) { + data, err := json.Marshal(patch) + if nil != err { + return existingGroupSnapshot, err + } + + newGroupSnapshot, err := client.GroupsnapshotV1alpha1().VolumeGroupSnapshots(existingGroupSnapshot.Namespace).Patch(context.TODO(), existingGroupSnapshot.Name, types.JSONPatchType, data, metav1.PatchOptions{}, subresources...) + if err != nil { + return existingGroupSnapshot, err + } + + return newGroupSnapshot, nil +} + // PatchVolumeGroupSnapshotContent patches a volume group snapshot content object func PatchVolumeGroupSnapshotContent( existingGroupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent, diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 9c89d16c7..4a3dcd88c 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -76,6 +76,10 @@ const ( VolumeSnapshotAsSourceFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-as-source-protection" // Name of finalizer on PVCs that is being used as a source to create VolumeSnapshots PVCFinalizer = "snapshot.storage.kubernetes.io/pvc-as-source-protection" + // Name of finalizer on VolumeGroupSnapshotContents that are bound by VolumeGroupSnapshots + VolumeGroupSnapshotContentFinalizer = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshotcontent-bound-protection" + // Name of finalizer on VolumeGroupSnapshots that are bound to VolumeGroupSnapshotContents + VolumeGroupSnapshotBoundFinalizer = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshot-bound-protection" IsDefaultSnapshotClassAnnotation = "snapshot.storage.kubernetes.io/is-default-class" IsDefaultGroupSnapshotClassAnnotation = "groupsnapshot.storage.kubernetes.io/is-default-class" @@ -114,6 +118,14 @@ const ( // group snapshots. AnnVolumeGroupSnapshotBeingCreated = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshot-being-created" + // AnnVolumeGroupSnapshotBeingDeleted annotation applies to VolumeGroupSnapshotContents. + // It indicates that the common snapshot controller has verified that volume + // group snapshot has a deletion timestamp and is being deleted. + // Sidecar controller needs to check the deletion policy on the + // VolumeGroupSnapshotContent and decide whether to delete the volume group snapshot + // backing the group snapshot content. + AnnVolumeGroupSnapshotBeingDeleted = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshot-being-deleted" + // Annotation for secret name and namespace will be added to the content // and used at snapshot content deletion time. AnnDeletionSecretRefName = "snapshot.storage.kubernetes.io/deletion-secret-name" @@ -406,12 +418,23 @@ func NeedToAddContentFinalizer(content *crdv1.VolumeSnapshotContent) bool { return content.ObjectMeta.DeletionTimestamp == nil && !ContainsString(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer) } +// NeedToAddGroupSnapshotContentFinalizer checks if a Finalizer needs to be added for the volume group snapshot content. +func NeedToAddGroupSnapshotContentFinalizer(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) bool { + return groupSnapshotContent.ObjectMeta.DeletionTimestamp == nil && !ContainsString(groupSnapshotContent.ObjectMeta.Finalizers, VolumeGroupSnapshotContentFinalizer) +} + // IsSnapshotDeletionCandidate checks if a volume snapshot deletionTimestamp // is set and any finalizer is on the snapshot. func IsSnapshotDeletionCandidate(snapshot *crdv1.VolumeSnapshot) bool { return snapshot.ObjectMeta.DeletionTimestamp != nil && (ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer) || ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer)) } +// IsGroupSnapshotDeletionCandidate checks if a volume group snapshot deletionTimestamp +// is set and any finalizer is on the group snapshot. +func IsGroupSnapshotDeletionCandidate(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) bool { + return groupSnapshot.ObjectMeta.DeletionTimestamp != nil && ContainsString(groupSnapshot.ObjectMeta.Finalizers, VolumeGroupSnapshotBoundFinalizer) +} + // NeedToAddSnapshotAsSourceFinalizer checks if a Finalizer needs to be added for the volume snapshot as a source for PVC. func NeedToAddSnapshotAsSourceFinalizer(snapshot *crdv1.VolumeSnapshot) bool { return snapshot.ObjectMeta.DeletionTimestamp == nil && !ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer) @@ -422,6 +445,11 @@ func NeedToAddSnapshotBoundFinalizer(snapshot *crdv1.VolumeSnapshot) bool { return snapshot.ObjectMeta.DeletionTimestamp == nil && !ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer) && IsBoundVolumeSnapshotContentNameSet(snapshot) } +// NeedToAddGroupSnapshotBoundFinalizer checks if a Finalizer needs to be added for the bound volume group snapshot. +func NeedToAddGroupSnapshotBoundFinalizer(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) bool { + return groupSnapshot.ObjectMeta.DeletionTimestamp == nil && !ContainsString(groupSnapshot.ObjectMeta.Finalizers, VolumeGroupSnapshotBoundFinalizer) && IsBoundVolumeGroupSnapshotContentNameSet(groupSnapshot) +} + func deprecationWarning(deprecatedParam, newParam, removalVersion string) string { if removalVersion == "" { removalVersion = "a future release" @@ -468,6 +496,18 @@ func GetSnapshotStatusForLogging(snapshot *crdv1.VolumeSnapshot) string { return fmt.Sprintf("bound to: %q, Completed: %v", snapshotContentName, ready) } +func GetGroupSnapshotStatusForLogging(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) string { + groupSnapshotContentName := "" + if groupSnapshot.Status != nil && groupSnapshot.Status.BoundVolumeGroupSnapshotContentName != nil { + groupSnapshotContentName = *groupSnapshot.Status.BoundVolumeGroupSnapshotContentName + } + ready := false + if groupSnapshot.Status != nil && groupSnapshot.Status.ReadyToUse != nil { + ready = *groupSnapshot.Status.ReadyToUse + } + return fmt.Sprintf("bound to: %q, Completed: %v", groupSnapshotContentName, ready) +} + func IsVolumeSnapshotRefSet(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool { if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace &&