fix: prevent MachineSetStatus from going into create/destroy loop

The `MachineSetStatus` resource getting modified in the finalizer removal function was causing it to go into a create/delete loop, filling up etcd. This issue is only observable when the `MachineSet` had the `MachineProvisionController` finalizer (when an infra provider is used).

Change the modify function to effectively be an update, never create the resource.

The fix is tested in a staging environment and confirmed to be resolving the issue.

Co-authored-by: Artem Chernyshev <[email protected]>
Signed-off-by: Utku Ozdemir <[email protected]>

Author: utkuozdemir

internal/backend/runtime/omni/controllers/omni/machine_set_status.go

@@ -7,6 +7,7 @@ package omni
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/cosi-project/runtime/pkg/controller"
@@ -213,7 +214,14 @@ func (handler *machineSetStatusHandler) reconcileTearingDown(ctx context.Context
 	}
 
 	updateStatus := func(clusterMachinesCount uint32) error {
-		return safe.WriterModify(ctx, r, omni.NewMachineSetStatus(resources.DefaultNamespace, machineSet.Metadata().ID()), func(status *omni.MachineSetStatus) error {
+		mss := omni.NewMachineSetStatus(resources.DefaultNamespace, machineSet.Metadata().ID())
+		notFoundErr := errors.New("not found")
+
+		modifyErr := safe.WriterModify(ctx, r, mss, func(status *omni.MachineSetStatus) error {
+			if status.Metadata().Version().Value() == 0 {
+				return notFoundErr
+			}
+
 			status.TypedSpec().Value.Phase = specs.MachineSetPhase_Destroying
 			status.TypedSpec().Value.Ready = false
 			status.TypedSpec().Value.Machines = &specs.Machines{
@@ -223,6 +231,11 @@ func (handler *machineSetStatusHandler) reconcileTearingDown(ctx context.Context
 
 			return nil
 		}, controller.WithExpectedPhaseAny())
+		if modifyErr != nil && !errors.Is(modifyErr, notFoundErr) {
+			return modifyErr
+		}
+
+		return nil
 	}
 
 	clusterMachinesCount := uint32(len(rc.GetClusterMachines()))