fix: remove dangling cluster taints

Because of an issue, after Cluster deletion, ClusterTaint resources were not cleaned up properly. This was discovered during the migration to move away from ClusterTaint resource. This commit will clean-up dangling taints.

Signed-off-by: Oguz Kilcan <[email protected]>
(cherry picked from commit ffd695f)

Author: oguzkilcan

internal/backend/runtime/omni/migration/migration_test.go

@@ -2199,10 +2199,14 @@ func (suite *MigrationSuite) TestMoveClusterTaintFromResourceToLabel() {
 	defer cancel()
 
 	clusterID := "cluster1"
+	deletedClusterID := "cluster2"
 
 	taint := omni.NewClusterTaint(resources.DefaultNamespace, clusterID)
 	suite.Require().NoError(suite.state.Create(ctx, taint))
 
+	danglingTaint := omni.NewClusterTaint(resources.DefaultNamespace, deletedClusterID)
+	suite.Require().NoError(suite.state.Create(ctx, danglingTaint))
+
 	clusterStatus := omni.NewClusterStatus(resources.DefaultNamespace, clusterID)
 	suite.Require().NoError(suite.state.Create(ctx, clusterStatus, state.WithCreateOwner(omnictrl.ClusterStatusControllerName)))
 
@@ -2221,6 +2225,11 @@ func (suite *MigrationSuite) TestMoveClusterTaintFromResourceToLabel() {
 	suite.Require().Error(err)
 	suite.Require().True(state.IsNotFoundError(err), "ClusterTaint resource should not exist after migration")
 	suite.Require().Nil(taintDeleted, "ClusterTaint resource should not exist after migration")
+
+	danglingTaintDeleted, err := safe.ReaderGetByID[*omni.ClusterTaint](ctx, suite.state, deletedClusterID)
+	suite.Require().Error(err)
+	suite.Require().True(state.IsNotFoundError(err), "ClusterTaint resource should not exist after migration")
+	suite.Require().Nil(danglingTaintDeleted, "ClusterTaint resource should not exist after migration")
 }
 
 func startMigration[

internal/backend/runtime/omni/migration/migrations.go

@@ -1931,9 +1931,10 @@ func moveClusterTaintFromResourceToLabel(ctx context.Context, st state.State, _
 
 			return nil
 		}, state.WithExpectedPhaseAny(), state.WithUpdateOwner(omnictrl.ClusterStatusControllerName))
-		if err != nil {
+		if err != nil && !state.IsNotFoundError(err) {
 			return err
 		}
+		// cluster status does not exist, taint is dangling, just remove it
 
 		if err = st.TeardownAndDestroy(ctx, taint.Metadata()); err != nil {
 			return err