diff --git a/cluster-autoscaler/core/scaledown/actuation/drain_test.go b/cluster-autoscaler/core/scaledown/actuation/drain_test.go index b3ca14805cf..bc49f46f6b5 100644 --- a/cluster-autoscaler/core/scaledown/actuation/drain_test.go +++ b/cluster-autoscaler/core/scaledown/actuation/drain_test.go @@ -35,6 +35,7 @@ import ( testprovider "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/test" "k8s.io/autoscaler/cluster-autoscaler/config" acontext "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/core/scaledown/status" . "k8s.io/autoscaler/cluster-autoscaler/core/test" "k8s.io/autoscaler/cluster-autoscaler/core/utils" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" @@ -365,6 +366,13 @@ func TestDrainNodeWithPodsWithRetries(t *testing.T) { assert.Equal(t, p3.Name, deleted[3]) } +type wantPodEvictionResult struct { + Pod *apiv1.Pod + Error error + TimedOut bool + Evicted bool +} + func TestDrainNodeWithPodsDaemonSetEvictionFailure(t *testing.T) { fakeClient := &fake.Clientset{} @@ -377,6 +385,11 @@ func TestDrainNodeWithPodsDaemonSetEvictionFailure(t *testing.T) { e1 := fmt.Errorf("eviction_error: d1") e2 := fmt.Errorf("eviction_error: d2") + wantEvictionResults := map[string]wantPodEvictionResult{ + "p1": {Pod: p1, Error: nil, TimedOut: false, Evicted: true}, + "p2": {Pod: p2, Error: nil, TimedOut: false, Evicted: true}, + } + fakeClient.Fake.AddReactor("get", "pods", func(action core.Action) (bool, runtime.Object, error) { return true, nil, errors.NewNotFound(apiv1.Resource("pod"), "whatever") }) @@ -416,15 +429,8 @@ func TestDrainNodeWithPodsDaemonSetEvictionFailure(t *testing.T) { assert.NoError(t, err) evictionResults, err := evictor.DrainNode(&ctx, nodeInfo) assert.NoError(t, err) - assert.Equal(t, 2, len(evictionResults)) - assert.Equal(t, p1, evictionResults["p1"].Pod) - assert.Equal(t, p2, evictionResults["p2"].Pod) - assert.NoError(t, evictionResults["p1"].Err) - assert.NoError(t, evictionResults["p2"].Err) - assert.False(t, evictionResults["p1"].TimedOut) - assert.False(t, evictionResults["p2"].TimedOut) - assert.True(t, evictionResults["p1"].WasEvictionSuccessful()) - assert.True(t, evictionResults["p2"].WasEvictionSuccessful()) + + assertPodEvictionResults(t, wantEvictionResults, evictionResults) } func TestDrainNodeWithPodsEvictionFailure(t *testing.T) { @@ -435,8 +441,17 @@ func TestDrainNodeWithPodsEvictionFailure(t *testing.T) { p2 := BuildTestPod("p2", 100, 0, WithNodeName(n1.Name)) p3 := BuildTestPod("p3", 100, 0, WithNodeName(n1.Name)) p4 := BuildTestPod("p4", 100, 0, WithNodeName(n1.Name)) + e2 := fmt.Errorf("eviction_error: p2") e4 := fmt.Errorf("eviction_error: p4") + + wantEvictionResults := map[string]wantPodEvictionResult{ + "p1": {Pod: p1, Error: nil, TimedOut: false, Evicted: true}, + "p2": {Pod: p2, Error: e2, TimedOut: true, Evicted: false}, + "p3": {Pod: p3, Error: nil, TimedOut: false, Evicted: true}, + "p4": {Pod: p4, Error: e4, TimedOut: true, Evicted: false}, + } + SetNodeReadyState(n1, true, time.Time{}) fakeClient.Fake.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { @@ -477,24 +492,9 @@ func TestDrainNodeWithPodsEvictionFailure(t *testing.T) { assert.NoError(t, err) evictionResults, err := evictor.DrainNode(&ctx, nodeInfo) assert.Error(t, err) - assert.Equal(t, 4, len(evictionResults)) - assert.Equal(t, *p1, *evictionResults["p1"].Pod) - assert.Equal(t, *p2, *evictionResults["p2"].Pod) - assert.Equal(t, *p3, *evictionResults["p3"].Pod) - assert.Equal(t, *p4, *evictionResults["p4"].Pod) - assert.NoError(t, evictionResults["p1"].Err) - assert.Contains(t, evictionResults["p2"].Err.Error(), e2.Error()) - assert.NoError(t, evictionResults["p3"].Err) - assert.Contains(t, evictionResults["p4"].Err.Error(), e4.Error()) - assert.False(t, evictionResults["p1"].TimedOut) - assert.True(t, evictionResults["p2"].TimedOut) - assert.False(t, evictionResults["p3"].TimedOut) - assert.True(t, evictionResults["p4"].TimedOut) - assert.True(t, evictionResults["p1"].WasEvictionSuccessful()) - assert.False(t, evictionResults["p2"].WasEvictionSuccessful()) - assert.True(t, evictionResults["p3"].WasEvictionSuccessful()) - assert.False(t, evictionResults["p4"].WasEvictionSuccessful()) - assert.Contains(t, r.pods, p1, p3) + + assertPodEvictionResults(t, wantEvictionResults, evictionResults) + assertEvictedPods(t, r.pods, evictionResults) } func TestDrainForceNodeWithPodsEvictionFailure(t *testing.T) { @@ -505,10 +505,18 @@ func TestDrainForceNodeWithPodsEvictionFailure(t *testing.T) { p2 := BuildTestPod("p2", 100, 0, WithNodeName(n1.Name)) p3 := BuildTestPod("p3", 100, 0, WithNodeName(n1.Name)) p4 := BuildTestPod("p4", 100, 0, WithNodeName(n1.Name)) + e2 := fmt.Errorf("eviction_error: p2") e4 := fmt.Errorf("eviction_error: p4") d4 := fmt.Errorf("deletion_error: p4") + wantEvictionResults := map[string]wantPodEvictionResult{ + "p1": {Pod: p1, Error: nil, TimedOut: false, Evicted: true}, + "p2": {Pod: p2, Error: nil, TimedOut: false, Evicted: true}, + "p3": {Pod: p3, Error: nil, TimedOut: false, Evicted: true}, + "p4": {Pod: p4, Error: d4, TimedOut: false, Evicted: false}, + } + SetNodeReadyState(n1, true, time.Time{}) fakeClient.Fake.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { @@ -566,24 +574,9 @@ func TestDrainForceNodeWithPodsEvictionFailure(t *testing.T) { assert.NoError(t, err) evictionResults, err := evictor.drainNodeForce(&ctx, nodeInfo) assert.Error(t, err) - assert.Equal(t, 4, len(evictionResults)) - assert.Equal(t, *p1, *evictionResults["p1"].Pod) - assert.Equal(t, *p2, *evictionResults["p2"].Pod) - assert.Equal(t, *p3, *evictionResults["p3"].Pod) - assert.Equal(t, *p4, *evictionResults["p4"].Pod) - assert.NoError(t, evictionResults["p1"].Err) - assert.NoError(t, evictionResults["p2"].Err) - assert.NoError(t, evictionResults["p3"].Err) - assert.Contains(t, evictionResults["p4"].Err.Error(), d4.Error()) - assert.False(t, evictionResults["p1"].TimedOut) - assert.False(t, evictionResults["p2"].TimedOut) - assert.False(t, evictionResults["p3"].TimedOut) - assert.False(t, evictionResults["p4"].TimedOut) - assert.True(t, evictionResults["p1"].WasEvictionSuccessful()) - assert.True(t, evictionResults["p2"].WasEvictionSuccessful()) - assert.True(t, evictionResults["p3"].WasEvictionSuccessful()) - assert.False(t, evictionResults["p4"].WasEvictionSuccessful()) - assert.Contains(t, r.pods, p1, p2, p3) + + assertPodEvictionResults(t, wantEvictionResults, evictionResults) + assertEvictedPods(t, r.pods, evictionResults) } func TestDrainWithPodsNodeDisappearanceFailure(t *testing.T) { @@ -594,7 +587,16 @@ func TestDrainWithPodsNodeDisappearanceFailure(t *testing.T) { p2 := BuildTestPod("p2", 100, 0, WithNodeName(n1.Name)) p3 := BuildTestPod("p3", 100, 0, WithNodeName(n1.Name)) p4 := BuildTestPod("p4", 100, 0, WithNodeName(n1.Name)) + e2 := fmt.Errorf("disappearance_error: p2") + + wantEvictionResults := map[string]wantPodEvictionResult{ + "p1": {Pod: p1, Error: nil, TimedOut: false, Evicted: true}, + "p2": {Pod: p2, Error: e2, TimedOut: true, Evicted: false}, + "p3": {Pod: p3, Error: nil, TimedOut: false, Evicted: true}, + "p4": {Pod: p4, Error: nil, TimedOut: true, Evicted: false}, + } + SetNodeReadyState(n1, true, time.Time{}) fakeClient.Fake.AddReactor("get", "pods", func(action core.Action) (bool, runtime.Object, error) { @@ -632,23 +634,35 @@ func TestDrainWithPodsNodeDisappearanceFailure(t *testing.T) { assert.NoError(t, err) evictionResults, err := evictor.DrainNode(&ctx, nodeInfo) assert.Error(t, err) - assert.Equal(t, 4, len(evictionResults)) - assert.Equal(t, *p1, *evictionResults["p1"].Pod) - assert.Equal(t, *p2, *evictionResults["p2"].Pod) - assert.Equal(t, *p3, *evictionResults["p3"].Pod) - assert.Equal(t, *p4, *evictionResults["p4"].Pod) - assert.NoError(t, evictionResults["p1"].Err) - assert.Contains(t, evictionResults["p2"].Err.Error(), e2.Error()) - assert.NoError(t, evictionResults["p3"].Err) - assert.NoError(t, evictionResults["p4"].Err) - assert.False(t, evictionResults["p1"].TimedOut) - assert.True(t, evictionResults["p2"].TimedOut) - assert.False(t, evictionResults["p3"].TimedOut) - assert.True(t, evictionResults["p4"].TimedOut) - assert.True(t, evictionResults["p1"].WasEvictionSuccessful()) - assert.False(t, evictionResults["p2"].WasEvictionSuccessful()) - assert.True(t, evictionResults["p3"].WasEvictionSuccessful()) - assert.False(t, evictionResults["p4"].WasEvictionSuccessful()) + + assertPodEvictionResults(t, wantEvictionResults, evictionResults) +} + +func assertPodEvictionResults(t *testing.T, wantEvictionResults map[string]wantPodEvictionResult, evictionResults map[string]status.PodEvictionResult) { + assert.Equal(t, len(wantEvictionResults), len(evictionResults)) + + for podName, want := range wantEvictionResults { + result := evictionResults[podName] + assert.Equal(t, *want.Pod, *result.Pod) + if want.Error == nil { + assert.NoError(t, result.Err) + } else { + assert.ErrorContains(t, result.Err, want.Error.Error()) + + } + assert.Equal(t, want.TimedOut, result.TimedOut) + assert.Equal(t, want.Evicted, result.WasEvictionSuccessful()) + } +} + +func assertEvictedPods(t *testing.T, wantEvictedPods []*apiv1.Pod, evictionResults map[string]status.PodEvictionResult) { + evicted := make([]*apiv1.Pod, 0) + for _, r := range evictionResults { + if r.WasEvictionSuccessful() { + evicted = append(evicted, r.Pod) + } + } + assert.ElementsMatch(t, wantEvictedPods, evicted) } func TestPodsToEvict(t *testing.T) {