controllers: fix decommission stuck when pod is deleted

When a Cassandra pod is deleted during decommission (e.g., StatefulSet
scaled down manually), casskop gets stuck in StatusOngoing state
because it tries to get a non-existent pod and returns an error
instead of transitioning to StatusFinalizing for PVC cleanup.

This fix handles the IsNotFound error case during StatusOngoing by
transitioning to StatusFinalizing and proceeding with PVC cleanup,
similar to how StatusFinalizing already handles this case.

The fix ensures casskop can properly recover when pods are deleted
during decommission operations, allowing the cluster to scale correctly.

Author: aignatov

controllers/cassandracluster/pod_operation.go

@@ -363,6 +363,20 @@ func (rcc *CassandraClusterReconciler) ensureDecommission(ctx context.Context, c
 
 		lastPod, err := rcc.GetPod(ctx, cc.Namespace, podLastOperation.Pods[0])
 		if err != nil {
+			// If pod is not found, it means it was already deleted (e.g., StatefulSet scaled down)
+			// Transition to StatusFinalizing to proceed with PVC cleanup
+			if apierrors.IsNotFound(err) {
+				logrus.WithFields(logrus.Fields{"cluster": cc.Name, "rack": dcRackName,
+					"pod": podLastOperation.Pods[0]}).Info("Pod not found during decommission, transitioning to Finalizing")
+				// Create a minimal pod object for deletePodPVC (it only needs the name)
+				lastPod = &v1.Pod{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      podLastOperation.Pods[0],
+						Namespace: cc.Namespace,
+					},
+				}
+				return rcc.deletePodPVC(ctx, cc, dcName, rackName, status, lastPod, statefulsetIsReady)
+			}
 			return breakResyncLoop, fmt.Errorf(
 				"failed to get last pod '%s': %v", podLastOperation.Pods[0], err)
 		}
@@ -640,8 +654,10 @@ func (rcc *CassandraClusterReconciler) updatePodLastOperation(clusterName, dcRac
 	podLastOperation.Pods = k8s.RemoveString(podLastOperation.Pods, podName)
 }
 
-/* finalizeOperation sets the labels on the pod where ran an operation depending on the error status
-   It also updates status.CassandraRackStatus[dcRackName].PodLastOperation
+/*
+finalizeOperation sets the labels on the pod where ran an operation depending on the error status
+
+	It also updates status.CassandraRackStatus[dcRackName].PodLastOperation
 */
 func (rcc *CassandraClusterReconciler) finalizeOperation(ctx context.Context, err error, cc *api.CassandraCluster, dcRackName string,
 	pod v1.Pod, status *api.CassandraClusterStatus, operationName string) {