From 35c87048c0a87338bb1ceae37cc2192cd3751a4c Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Tue, 24 Sep 2024 13:43:12 +0300 Subject: [PATCH 1/7] cluster-up: Add nse flag to OVNK kind script This will enable the network-segmentation feature on OVNK, needed in order to run VM workloads with primary UDN. Signed-off-by: Ram Lavi --- hack/cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/cluster.sh b/hack/cluster.sh index 3a7fa27a..d7b7a698 100755 --- a/hack/cluster.sh +++ b/hack/cluster.sh @@ -3,7 +3,7 @@ set -xe SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -KIND_ARGS="${KIND_ARGS:--ic -ikv -i6 -mne}" +KIND_ARGS="${KIND_ARGS:--ic -ikv -i6 -mne -nse}" OUTPUT_DIR=${OUTPUT_DIR:-${SCRIPT_DIR}/../.output} From da4651d309f4f9b6d517d96c2aac3f2af9c447d2 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Tue, 24 Sep 2024 13:15:53 +0300 Subject: [PATCH 2/7] tests/e2e, persistentips: Add role to GenerateLayer2WithSubnetNAD Currently GenerateLayer2WithSubnetNAD is creating NADs for secondary roles (which is the default). Add role input param to the function in order to support primary role that will be used in future commits. Signed-off-by: Ram Lavi --- test/e2e/persistentips_test.go | 3 ++- test/env/generate.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e/persistentips_test.go b/test/e2e/persistentips_test.go index 081f31c0..4266e0de 100644 --- a/test/e2e/persistentips_test.go +++ b/test/e2e/persistentips_test.go @@ -58,6 +58,7 @@ var _ = Describe("Persistent IPs", func() { When("network attachment definition created with allowPersistentIPs=true", func() { var ( td testenv.TestData + role = "secondary" networkInterfaceName = "multus" vm *kubevirtv1.VirtualMachine vmi *kubevirtv1.VirtualMachineInstance @@ -70,7 +71,7 @@ var _ = Describe("Persistent IPs", func() { td.TearDown() }) - nad = testenv.GenerateLayer2WithSubnetNAD(td.Namespace) + nad = testenv.GenerateLayer2WithSubnetNAD(td.Namespace, role) vmi = testenv.GenerateAlpineWithMultusVMI(td.Namespace, networkInterfaceName, nad.Name) vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning()) diff --git a/test/env/generate.go b/test/env/generate.go index 66813054..c6d8878d 100644 --- a/test/env/generate.go +++ b/test/env/generate.go @@ -13,7 +13,7 @@ import ( kubevirtv1 "kubevirt.io/api/core/v1" ) -func GenerateLayer2WithSubnetNAD(namespace string) *nadv1.NetworkAttachmentDefinition { +func GenerateLayer2WithSubnetNAD(namespace, role string) *nadv1.NetworkAttachmentDefinition { networkName := "l2" nadName := RandomName(networkName, 16) return &nadv1.NetworkAttachmentDefinition{ @@ -30,9 +30,10 @@ func GenerateLayer2WithSubnetNAD(namespace string) *nadv1.NetworkAttachmentDefin "topology": "layer2", "subnets": "10.100.200.0/24", "netAttachDefName": "%[1]s/%[2]s", + "role": "%[4]s", "allowPersistentIPs": true } -`, namespace, nadName, networkName), +`, namespace, nadName, networkName, role), }, } } From b119fa5a9c1e176cc3c4f5a87b5c865ba378485f Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Tue, 1 Oct 2024 11:59:47 +0300 Subject: [PATCH 3/7] tests/e2e, persistentip: Move GetIPs to general function In the current tests (where we only run persistent ip tests for secondary interfaces), the ips of the VMI is extracted via vmi.status. This is done both directly and via the matchers in testenv library. In order for the test suite to support running the same tests on primary UDN VMs, this function needs to be generalized in the following ways: 1. it should align all the IP retrievals to use this new function (right now the IPs are retrieved both directly accessing vmi.status and via a function). 2. it should move the function retrieving the IPs to a first-class functions variable, so that the way IPS are retrieved can be generalized. 3. it should only have the common parameters needed to get the IPs (which is the VMI object). Moving get the IP extraction to a general function, so that it could be consumed uniformly. Doing this will allow to more easily retrieve the IP in other cases such as primary UDN, where the IP is currently retrieved in another way. This will be added in future commits. Signed-off-by: Ram Lavi --- test/e2e/persistentips_test.go | 57 ++++++++++++++++++---------------- test/env/getter.go | 20 +++++++++++- test/env/matcher.go | 33 ++++---------------- 3 files changed, 56 insertions(+), 54 deletions(-) diff --git a/test/e2e/persistentips_test.go b/test/e2e/persistentips_test.go index 4266e0de..b8affec3 100644 --- a/test/e2e/persistentips_test.go +++ b/test/e2e/persistentips_test.go @@ -40,6 +40,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const secondaryLogicalNetworkInterfaceName = "multus" + var _ = Describe("Persistent IPs", func() { var failureCount int = 0 JustAfterEach(func() { @@ -57,13 +59,15 @@ var _ = Describe("Persistent IPs", func() { When("network attachment definition created with allowPersistentIPs=true", func() { var ( - td testenv.TestData - role = "secondary" - networkInterfaceName = "multus" - vm *kubevirtv1.VirtualMachine - vmi *kubevirtv1.VirtualMachineInstance - nad *nadv1.NetworkAttachmentDefinition + td testenv.TestData + ipsFrom func(vmi *kubevirtv1.VirtualMachineInstance) []string + role = "secondary" + vm *kubevirtv1.VirtualMachine + vmi *kubevirtv1.VirtualMachineInstance + nad *nadv1.NetworkAttachmentDefinition ) + + ipsFrom = secondaryNetworkVMIStatusIPs BeforeEach(func() { td = testenv.GenerateTestData() td.SetUp() @@ -72,7 +76,7 @@ var _ = Describe("Persistent IPs", func() { }) nad = testenv.GenerateLayer2WithSubnetNAD(td.Namespace, role) - vmi = testenv.GenerateAlpineWithMultusVMI(td.Namespace, networkInterfaceName, nad.Name) + vmi = testenv.GenerateAlpineWithMultusVMI(td.Namespace, secondaryLogicalNetworkInterfaceName, nad.Name) vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning()) By("Create NetworkAttachmentDefinition") @@ -95,14 +99,13 @@ var _ = Describe("Persistent IPs", func() { WithPolling(time.Second). ShouldNot(BeEmpty()) - Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - - Expect(vmi.Status.Interfaces).NotTo(BeEmpty()) - Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty()) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, Not(BeEmpty()))) }) It("should keep ips after live migration", func() { - vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs + Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) + vmiIPsBeforeMigration := ipsFrom(vmi) + Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vm.Name) testenv.CheckLiveMigrationSucceeded(td.Namespace, vm.Name) @@ -113,8 +116,7 @@ var _ = Describe("Persistent IPs", func() { WithTimeout(5 * time.Minute). Should(testenv.ContainConditionVMIReady()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration))) - + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, ConsistOf(vmiIPsBeforeMigration))) }) It("should garbage collect IPAMClaims after VM deletion", func() { @@ -172,7 +174,9 @@ var _ = Describe("Persistent IPs", func() { }) It("should keep ips after restart", func() { - vmiIPsBeforeRestart := vmi.Status.Interfaces[0].IPs + Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) + vmiIPsBeforeRestart := ipsFrom(vmi) + Expect(vmiIPsBeforeRestart).NotTo(BeEmpty()) vmiUUIDBeforeRestart := vmi.UID By("Re-starting the VM") @@ -191,7 +195,7 @@ var _ = Describe("Persistent IPs", func() { WithTimeout(5 * time.Minute). Should(testenv.ContainConditionVMIReady()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeRestart))) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, ConsistOf(vmiIPsBeforeRestart))) }) }) @@ -218,9 +222,8 @@ var _ = Describe("Persistent IPs", func() { ShouldNot(BeEmpty()) Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - - Expect(vmi.Status.Interfaces).NotTo(BeEmpty()) - Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty()) + ips := ipsFrom(vmi) + Expect(ips).NotTo(BeEmpty()) }) It("should garbage collect IPAMClaims after VM foreground deletion, only after VMI is gone", func() { @@ -261,20 +264,18 @@ var _ = Describe("Persistent IPs", func() { WithPolling(time.Second). ShouldNot(BeEmpty()) - Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - - Expect(vmi.Status.Interfaces).NotTo(BeEmpty()) - Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty()) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, Not(BeEmpty()))) }) It("should keep ips after live migration", func() { - vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs + Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) + vmiIPsBeforeMigration := ipsFrom(vmi) + Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vmi.Name) testenv.CheckLiveMigrationSucceeded(td.Namespace, vmi.Name) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration))) - + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, ConsistOf(vmiIPsBeforeMigration))) }) It("should garbage collect IPAMClaims after VMI deletion", func() { @@ -312,3 +313,7 @@ func removeFinalizersPatch() ([]byte, error) { } return json.Marshal(patch) } + +func secondaryNetworkVMIStatusIPs(vmi *kubevirtv1.VirtualMachineInstance) []string { + return testenv.GetIPsFromVMIStatus(vmi, secondaryLogicalNetworkInterfaceName) +} diff --git a/test/env/getter.go b/test/env/getter.go index 7c620cc9..c95bde35 100644 --- a/test/env/getter.go +++ b/test/env/getter.go @@ -4,8 +4,9 @@ import ( "context" apierrors "k8s.io/apimachinery/pkg/api/errors" - kubevirtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + kubevirtv1 "kubevirt.io/api/core/v1" ) // ThisVMI fetches the latest state of the VirtualMachineInstance. If the object does not exist, nil is returned. @@ -28,3 +29,20 @@ func ThisVMReadiness(vm *kubevirtv1.VirtualMachine) func() (bool, error) { return vm.Status.Ready, nil } } + +func lookupInterfaceStatusByName(interfaces []kubevirtv1.VirtualMachineInstanceNetworkInterface, name string) *kubevirtv1.VirtualMachineInstanceNetworkInterface { + for index := range interfaces { + if interfaces[index].Name == name { + return &interfaces[index] + } + } + return nil +} + +func GetIPsFromVMIStatus(vmi *kubevirtv1.VirtualMachineInstance, networkInterfaceName string) []string { + ifaceStatus := lookupInterfaceStatusByName(vmi.Status.Interfaces, networkInterfaceName) + if ifaceStatus == nil { + return nil + } + return ifaceStatus.IPs +} diff --git a/test/env/matcher.go b/test/env/matcher.go index ec789ce6..7cacdc36 100644 --- a/test/env/matcher.go +++ b/test/env/matcher.go @@ -28,37 +28,16 @@ func IPAMClaimsFromNamespace(namespace string) func() ([]ipamclaimsv1alpha1.IPAM } } -func lookupInterfaceStatusByName(interfaces []kubevirtv1.VirtualMachineInstanceNetworkInterface, name string) *kubevirtv1.VirtualMachineInstanceNetworkInterface { - for index := range interfaces { - if interfaces[index].Name == name { - return &interfaces[index] - } - } - return nil -} - -func VMIStatusInterfaces(vmi *kubevirtv1.VirtualMachineInstance) []kubevirtv1.VirtualMachineInstanceNetworkInterface { - return vmi.Status.Interfaces -} func vmiStatusConditions(vmi *kubevirtv1.VirtualMachineInstance) []kubevirtv1.VirtualMachineInstanceCondition { return vmi.Status.Conditions } -func interfaceIPs(networkInterface *kubevirtv1.VirtualMachineInstanceNetworkInterface) []string { - if networkInterface == nil { - return nil - } - return networkInterface.IPs -} - -func MatchIPsAtInterfaceByName(interfaceName string, ipsMatcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { - return WithTransform( - func(vmi *kubevirtv1.VirtualMachineInstance) *kubevirtv1.VirtualMachineInstanceNetworkInterface { - return lookupInterfaceStatusByName(vmi.Status.Interfaces, interfaceName) - }, - SatisfyAll( - Not(BeNil()), - WithTransform(interfaceIPs, ipsMatcher))) +func MatchIPs(getIPsFunc func(vmi *kubevirtv1.VirtualMachineInstance) []string, ipsMatcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { + return WithTransform(func(vmi *kubevirtv1.VirtualMachineInstance) []string { + return getIPsFunc(vmi) + }, SatisfyAll( + WithTransform(func(ips []string) []string { return ips }, ipsMatcher), + )) } func BeRestarted(oldUID types.UID) gomegatypes.GomegaMatcher { From f007a5e819a91a1f446928e752242e320ea76bcc Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Mon, 7 Oct 2024 14:29:23 +0300 Subject: [PATCH 4/7] tests/e2e, persistentip: Move generateVMI to general function In the current tests (where we only run persistent ip tests for secondary interfaces), the VMI is created using the GenerateAlpineWithMultusVMI function. In order for the test suite to support running the same tests on primary UDN VMIs, refactoring the function to use VMI factory function. Moreover, the NAD Name is changed to be static, as it allows it to be consumed before runtime, as a test parameter for both the NAD and VMI creation. This shouldn't interfere with the tests operations as the NADs are created in different namespaces. Signed-off-by: Ram Lavi --- test/e2e/persistentips_test.go | 32 ++++++++++++++++-- test/env/generate.go | 60 +++++++++++++++++++--------------- 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/test/e2e/persistentips_test.go b/test/e2e/persistentips_test.go index b8affec3..bfa9b8be 100644 --- a/test/e2e/persistentips_test.go +++ b/test/e2e/persistentips_test.go @@ -40,7 +40,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const secondaryLogicalNetworkInterfaceName = "multus" +const ( + secondaryLogicalNetworkInterfaceName = "multus" + nadName = "l2-net-attach-def" +) var _ = Describe("Persistent IPs", func() { var failureCount int = 0 @@ -75,8 +78,8 @@ var _ = Describe("Persistent IPs", func() { td.TearDown() }) - nad = testenv.GenerateLayer2WithSubnetNAD(td.Namespace, role) - vmi = testenv.GenerateAlpineWithMultusVMI(td.Namespace, secondaryLogicalNetworkInterfaceName, nad.Name) + nad = testenv.GenerateLayer2WithSubnetNAD(nadName, td.Namespace, role) + vmi = vmiWithMultus(td.Namespace) vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning()) By("Create NetworkAttachmentDefinition") @@ -317,3 +320,26 @@ func removeFinalizersPatch() ([]byte, error) { func secondaryNetworkVMIStatusIPs(vmi *kubevirtv1.VirtualMachineInstance) []string { return testenv.GetIPsFromVMIStatus(vmi, secondaryLogicalNetworkInterfaceName) } + +func vmiWithMultus(namespace string) *kubevirtv1.VirtualMachineInstance { + interfaceName := secondaryLogicalNetworkInterfaceName + return testenv.NewVirtualMachineInstance( + namespace, + testenv.WithMemory("128Mi"), + testenv.WithInterface(kubevirtv1.Interface{ + Name: interfaceName, + InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{ + Bridge: &kubevirtv1.InterfaceBridge{}, + }, + }), + testenv.WithNetwork(kubevirtv1.Network{ + + Name: interfaceName, + NetworkSource: kubevirtv1.NetworkSource{ + Multus: &kubevirtv1.MultusNetwork{ + NetworkName: nadName, + }, + }, + }), + ) +} diff --git a/test/env/generate.go b/test/env/generate.go index c6d8878d..a4a9bbc2 100644 --- a/test/env/generate.go +++ b/test/env/generate.go @@ -13,9 +13,8 @@ import ( kubevirtv1 "kubevirt.io/api/core/v1" ) -func GenerateLayer2WithSubnetNAD(namespace, role string) *nadv1.NetworkAttachmentDefinition { +func GenerateLayer2WithSubnetNAD(nadName, namespace, role string) *nadv1.NetworkAttachmentDefinition { networkName := "l2" - nadName := RandomName(networkName, 16) return &nadv1.NetworkAttachmentDefinition{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -38,19 +37,16 @@ func GenerateLayer2WithSubnetNAD(namespace, role string) *nadv1.NetworkAttachmen } } -func GenerateAlpineWithMultusVMI(namespace, interfaceName, networkName string) *kubevirtv1.VirtualMachineInstance { - return &kubevirtv1.VirtualMachineInstance{ +type VMIOption func(vmi *kubevirtv1.VirtualMachineInstance) + +func NewVirtualMachineInstance(namespace string, opts ...VMIOption) *kubevirtv1.VirtualMachineInstance { + vmi := &kubevirtv1.VirtualMachineInstance{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: RandomName("alpine", 16), }, Spec: kubevirtv1.VirtualMachineInstanceSpec{ Domain: kubevirtv1.DomainSpec{ - Resources: kubevirtv1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - }, Devices: kubevirtv1.Devices{ Disks: []kubevirtv1.Disk{ { @@ -62,26 +58,10 @@ func GenerateAlpineWithMultusVMI(namespace, interfaceName, networkName string) * Name: "containerdisk", }, }, - Interfaces: []kubevirtv1.Interface{ - { - Name: interfaceName, - InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{ - Bridge: &kubevirtv1.InterfaceBridge{}, - }, - }, - }, - }, - }, - Networks: []kubevirtv1.Network{ - { - Name: interfaceName, - NetworkSource: kubevirtv1.NetworkSource{ - Multus: &kubevirtv1.MultusNetwork{ - NetworkName: networkName, - }, - }, + Interfaces: []kubevirtv1.Interface{}, }, }, + Networks: []kubevirtv1.Network{}, TerminationGracePeriodSeconds: pointer.Int64(5), Volumes: []kubevirtv1.Volume{ { @@ -95,6 +75,32 @@ func GenerateAlpineWithMultusVMI(namespace, interfaceName, networkName string) * }, }, } + + for _, f := range opts { + f(vmi) + } + + return vmi +} + +func WithMemory(memory string) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Domain.Resources.Requests = corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(memory), + } + } +} + +func WithInterface(iface kubevirtv1.Interface) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Domain.Devices.Interfaces = append(vmi.Spec.Domain.Devices.Interfaces, iface) + } +} + +func WithNetwork(network kubevirtv1.Network) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Networks = append(vmi.Spec.Networks, network) + } } type VMOption func(vm *kubevirtv1.VirtualMachine) From 9505683000656430459ce5d6e8e222bcb98c4359 Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Mon, 7 Oct 2024 14:35:02 +0300 Subject: [PATCH 5/7] tests/e2e, persistentip: Expand test matric using DescribeTableSubtree Expand the current test to run in a DescribeTableSubtree context. This is in preparation of adding more interface types in future commits. This commit does not add tests, nor change their operation. Signed-off-by: Ram Lavi --- test/e2e/persistentips_test.go | 54 +++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/test/e2e/persistentips_test.go b/test/e2e/persistentips_test.go index bfa9b8be..c411a71c 100644 --- a/test/e2e/persistentips_test.go +++ b/test/e2e/persistentips_test.go @@ -45,7 +45,17 @@ const ( nadName = "l2-net-attach-def" ) -var _ = Describe("Persistent IPs", func() { +const ( + roleSecondary = "secondary" +) + +type testParams struct { + role string + ipsFrom func(vmi *kubevirtv1.VirtualMachineInstance) []string + vmi func(namespace string) *kubevirtv1.VirtualMachineInstance +} + +var _ = DescribeTableSubtree("Persistent IPs", func(params testParams) { var failureCount int = 0 JustAfterEach(func() { if CurrentSpecReport().Failed() { @@ -62,15 +72,12 @@ var _ = Describe("Persistent IPs", func() { When("network attachment definition created with allowPersistentIPs=true", func() { var ( - td testenv.TestData - ipsFrom func(vmi *kubevirtv1.VirtualMachineInstance) []string - role = "secondary" - vm *kubevirtv1.VirtualMachine - vmi *kubevirtv1.VirtualMachineInstance - nad *nadv1.NetworkAttachmentDefinition + td testenv.TestData + vm *kubevirtv1.VirtualMachine + vmi *kubevirtv1.VirtualMachineInstance + nad *nadv1.NetworkAttachmentDefinition ) - ipsFrom = secondaryNetworkVMIStatusIPs BeforeEach(func() { td = testenv.GenerateTestData() td.SetUp() @@ -78,8 +85,8 @@ var _ = Describe("Persistent IPs", func() { td.TearDown() }) - nad = testenv.GenerateLayer2WithSubnetNAD(nadName, td.Namespace, role) - vmi = vmiWithMultus(td.Namespace) + nad = testenv.GenerateLayer2WithSubnetNAD(nadName, td.Namespace, params.role) + vmi = params.vmi(td.Namespace) vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning()) By("Create NetworkAttachmentDefinition") @@ -102,12 +109,12 @@ var _ = Describe("Persistent IPs", func() { WithPolling(time.Second). ShouldNot(BeEmpty()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, Not(BeEmpty()))) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, Not(BeEmpty()))) }) It("should keep ips after live migration", func() { Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - vmiIPsBeforeMigration := ipsFrom(vmi) + vmiIPsBeforeMigration := params.ipsFrom(vmi) Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vm.Name) @@ -119,7 +126,7 @@ var _ = Describe("Persistent IPs", func() { WithTimeout(5 * time.Minute). Should(testenv.ContainConditionVMIReady()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, ConsistOf(vmiIPsBeforeMigration))) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, ConsistOf(vmiIPsBeforeMigration))) }) It("should garbage collect IPAMClaims after VM deletion", func() { @@ -178,7 +185,7 @@ var _ = Describe("Persistent IPs", func() { It("should keep ips after restart", func() { Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - vmiIPsBeforeRestart := ipsFrom(vmi) + vmiIPsBeforeRestart := params.ipsFrom(vmi) Expect(vmiIPsBeforeRestart).NotTo(BeEmpty()) vmiUUIDBeforeRestart := vmi.UID @@ -198,7 +205,7 @@ var _ = Describe("Persistent IPs", func() { WithTimeout(5 * time.Minute). Should(testenv.ContainConditionVMIReady()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, ConsistOf(vmiIPsBeforeRestart))) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, ConsistOf(vmiIPsBeforeRestart))) }) }) @@ -225,7 +232,7 @@ var _ = Describe("Persistent IPs", func() { ShouldNot(BeEmpty()) Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - ips := ipsFrom(vmi) + ips := params.ipsFrom(vmi) Expect(ips).NotTo(BeEmpty()) }) @@ -267,18 +274,18 @@ var _ = Describe("Persistent IPs", func() { WithPolling(time.Second). ShouldNot(BeEmpty()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, Not(BeEmpty()))) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, Not(BeEmpty()))) }) It("should keep ips after live migration", func() { Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - vmiIPsBeforeMigration := ipsFrom(vmi) + vmiIPsBeforeMigration := params.ipsFrom(vmi) Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vmi.Name) testenv.CheckLiveMigrationSucceeded(td.Namespace, vmi.Name) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(ipsFrom, ConsistOf(vmiIPsBeforeMigration))) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, ConsistOf(vmiIPsBeforeMigration))) }) It("should garbage collect IPAMClaims after VMI deletion", func() { @@ -299,7 +306,14 @@ var _ = Describe("Persistent IPs", func() { }) }) -}) +}, + Entry("secondary interfaces", + testParams{ + role: roleSecondary, + ipsFrom: secondaryNetworkVMIStatusIPs, + vmi: vmiWithMultus, + }), +) func foregroundDeleteOptions() *client.DeleteOptions { foreground := metav1.DeletePropagationForeground From 8c58962321ed4c43a9d31cccdc54acecde8a964c Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Tue, 8 Oct 2024 22:11:20 +0300 Subject: [PATCH 6/7] tests/e2e, persistentip: Introduce primary-UDN helper functions Introducing the functions needed in order to run the current test suite for primary UDN interfaces: - defaultNetworkStatusAnnotationIPs - as the getIP function for primary UDN. - vmiWithPasst - as the vmi Generating function for primary UDN. These will be introduced as parameter functions when primary-UDN Entry is introduced in future commit. Moreover, since the function getting the IPs for primary-UDN case needs to return error, this is added to the general function signature. Signed-off-by: Ram Lavi --- test/e2e/persistentips_test.go | 55 ++++++++++++++++++++---- test/env/generate.go | 13 ++++++ test/env/getter.go | 77 ++++++++++++++++++++++++++++++++++ test/env/matcher.go | 15 +++++-- 4 files changed, 149 insertions(+), 11 deletions(-) diff --git a/test/e2e/persistentips_test.go b/test/e2e/persistentips_test.go index c411a71c..5cbd7bbc 100644 --- a/test/e2e/persistentips_test.go +++ b/test/e2e/persistentips_test.go @@ -51,7 +51,7 @@ const ( type testParams struct { role string - ipsFrom func(vmi *kubevirtv1.VirtualMachineInstance) []string + ipsFrom func(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error) vmi func(namespace string) *kubevirtv1.VirtualMachineInstance } @@ -114,7 +114,8 @@ var _ = DescribeTableSubtree("Persistent IPs", func(params testParams) { It("should keep ips after live migration", func() { Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - vmiIPsBeforeMigration := params.ipsFrom(vmi) + vmiIPsBeforeMigration, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vm.Name) @@ -185,7 +186,8 @@ var _ = DescribeTableSubtree("Persistent IPs", func(params testParams) { It("should keep ips after restart", func() { Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - vmiIPsBeforeRestart := params.ipsFrom(vmi) + vmiIPsBeforeRestart, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) Expect(vmiIPsBeforeRestart).NotTo(BeEmpty()) vmiUUIDBeforeRestart := vmi.UID @@ -232,7 +234,8 @@ var _ = DescribeTableSubtree("Persistent IPs", func(params testParams) { ShouldNot(BeEmpty()) Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - ips := params.ipsFrom(vmi) + ips, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) Expect(ips).NotTo(BeEmpty()) }) @@ -279,7 +282,8 @@ var _ = DescribeTableSubtree("Persistent IPs", func(params testParams) { It("should keep ips after live migration", func() { Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - vmiIPsBeforeMigration := params.ipsFrom(vmi) + vmiIPsBeforeMigration, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vmi.Name) @@ -331,8 +335,17 @@ func removeFinalizersPatch() ([]byte, error) { return json.Marshal(patch) } -func secondaryNetworkVMIStatusIPs(vmi *kubevirtv1.VirtualMachineInstance) []string { - return testenv.GetIPsFromVMIStatus(vmi, secondaryLogicalNetworkInterfaceName) +func secondaryNetworkVMIStatusIPs(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error) { + return testenv.GetIPsFromVMIStatus(vmi, secondaryLogicalNetworkInterfaceName), nil +} + +func defaultNetworkStatusAnnotationIPs(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error) { + defNetworkStatus, err := testenv.DefaultNetworkStatus(vmi) + if err != nil { + return nil, err + } + + return defNetworkStatus.IPs, nil } func vmiWithMultus(namespace string) *kubevirtv1.VirtualMachineInstance { @@ -357,3 +370,31 @@ func vmiWithMultus(namespace string) *kubevirtv1.VirtualMachineInstance { }), ) } + +func vmiWithPasst(namespace string) *kubevirtv1.VirtualMachineInstance { + const ( + interfaceName = "passtnet" + cloudInitNetworkData = ` +version: 2 +ethernets: + eth0: + dhcp4: true` + ) + return testenv.NewVirtualMachineInstance( + namespace, + testenv.WithMemory("2048Mi"), + testenv.WithInterface(kubevirtv1.Interface{ + Name: interfaceName, + Binding: &kubevirtv1.PluginBinding{ + Name: "passt", + }, + }), + testenv.WithNetwork(kubevirtv1.Network{ + Name: interfaceName, + NetworkSource: kubevirtv1.NetworkSource{ + Pod: &kubevirtv1.PodNetwork{}, + }, + }), + testenv.WithCloudInitNoCloudVolume(cloudInitNetworkData), + ) +} diff --git a/test/env/generate.go b/test/env/generate.go index a4a9bbc2..c5440c70 100644 --- a/test/env/generate.go +++ b/test/env/generate.go @@ -103,6 +103,19 @@ func WithNetwork(network kubevirtv1.Network) VMIOption { } } +func WithCloudInitNoCloudVolume(cloudInitNetworkData string) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Volumes = append(vmi.Spec.Volumes, kubevirtv1.Volume{ + Name: "cloudinitdisk", + VolumeSource: kubevirtv1.VolumeSource{ + CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ + NetworkData: cloudInitNetworkData, + }, + }, + }) + } +} + type VMOption func(vm *kubevirtv1.VirtualMachine) func NewVirtualMachine(vmi *kubevirtv1.VirtualMachineInstance, opts ...VMOption) *kubevirtv1.VirtualMachine { diff --git a/test/env/getter.go b/test/env/getter.go index c95bde35..4f6f52b6 100644 --- a/test/env/getter.go +++ b/test/env/getter.go @@ -2,7 +2,12 @@ package env import ( "context" + "encoding/json" + "fmt" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" @@ -46,3 +51,75 @@ func GetIPsFromVMIStatus(vmi *kubevirtv1.VirtualMachineInstance, networkInterfac } return ifaceStatus.IPs } + +func virtualMachineInstancePod(vmi *kubevirtv1.VirtualMachineInstance) (*corev1.Pod, error) { + pod, err := lookupPodBySelector(vmi.Namespace, vmiLabelSelector(vmi), vmiFieldSelector(vmi)) + if err != nil { + return nil, fmt.Errorf("failed to find pod for VMI %s (%s)", vmi.Name, string(vmi.GetUID())) + } + return pod, nil +} + +func lookupPodBySelector(namespace string, labelSelector, fieldSelector map[string]string) (*corev1.Pod, error) { + pods := &corev1.PodList{} + err := Client.List( + context.Background(), + pods, + client.InNamespace(namespace), + client.MatchingLabels(labelSelector), + client.MatchingFields(fieldSelector)) + if err != nil { + return nil, err + } + + if len(pods.Items) == 0 { + return nil, fmt.Errorf("failed to lookup pod with labels %v, fields %v in namespace %s", labelSelector, fieldSelector, namespace) + } + + return &pods.Items[0], nil +} + +func vmiLabelSelector(vmi *kubevirtv1.VirtualMachineInstance) map[string]string { + return map[string]string{kubevirtv1.CreatedByLabel: string(vmi.GetUID())} +} + +func vmiFieldSelector(vmi *kubevirtv1.VirtualMachineInstance) map[string]string { + fieldSelectors := map[string]string{} + if vmi.Status.Phase == kubevirtv1.Running { + const podPhase = "status.phase" + fieldSelectors[podPhase] = string(corev1.PodRunning) + } + return fieldSelectors +} + +func parsePodNetworkStatusAnnotation(podNetStatus string) ([]nadv1.NetworkStatus, error) { + if len(podNetStatus) == 0 { + return nil, fmt.Errorf("network status annotation not found") + } + + var netStatus []nadv1.NetworkStatus + if err := json.Unmarshal([]byte(podNetStatus), &netStatus); err != nil { + return nil, err + } + + return netStatus, nil +} + +func DefaultNetworkStatus(vmi *kubevirtv1.VirtualMachineInstance) (*nadv1.NetworkStatus, error) { + virtLauncherPod, err := virtualMachineInstancePod(vmi) + if err != nil { + return nil, err + } + + netStatuses, err := parsePodNetworkStatusAnnotation(virtLauncherPod.Annotations[nadv1.NetworkStatusAnnot]) + if err != nil { + return nil, err + } + + for _, netStatus := range netStatuses { + if netStatus.Default { + return &netStatus, nil + } + } + return nil, fmt.Errorf("primary IPs not found") +} diff --git a/test/env/matcher.go b/test/env/matcher.go index 7cacdc36..c7654b54 100644 --- a/test/env/matcher.go +++ b/test/env/matcher.go @@ -32,11 +32,18 @@ func vmiStatusConditions(vmi *kubevirtv1.VirtualMachineInstance) []kubevirtv1.Vi return vmi.Status.Conditions } -func MatchIPs(getIPsFunc func(vmi *kubevirtv1.VirtualMachineInstance) []string, ipsMatcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { - return WithTransform(func(vmi *kubevirtv1.VirtualMachineInstance) []string { - return getIPsFunc(vmi) +type IPResult struct { + IPs []string + Err error +} + +func MatchIPs(getIPsFunc func(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error), ipsMatcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { + return WithTransform(func(vmi *kubevirtv1.VirtualMachineInstance) IPResult { + ips, err := getIPsFunc(vmi) + return IPResult{IPs: ips, Err: err} }, SatisfyAll( - WithTransform(func(ips []string) []string { return ips }, ipsMatcher), + WithTransform(func(result IPResult) error { return result.Err }, Succeed()), + WithTransform(func(result IPResult) []string { return result.IPs }, ipsMatcher), )) } From a873dfb189486ae50f66552362a8811adb6e587f Mon Sep 17 00:00:00 2001 From: Ram Lavi Date: Mon, 7 Oct 2024 14:47:15 +0300 Subject: [PATCH 7/7] tests/e2e, persistentip: Add primary UDN test case Add a new test subtree entry for primary UDN interfaces, checking persistentIPs on workload with primary UDN. Signed-off-by: Ram Lavi --- test/e2e/persistentips_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/e2e/persistentips_test.go b/test/e2e/persistentips_test.go index 5cbd7bbc..738f8d53 100644 --- a/test/e2e/persistentips_test.go +++ b/test/e2e/persistentips_test.go @@ -46,6 +46,7 @@ const ( ) const ( + rolePrimary = "primary" roleSecondary = "secondary" ) @@ -317,6 +318,12 @@ var _ = DescribeTableSubtree("Persistent IPs", func(params testParams) { ipsFrom: secondaryNetworkVMIStatusIPs, vmi: vmiWithMultus, }), + Entry("primary UDN", + testParams{ + role: rolePrimary, + ipsFrom: defaultNetworkStatusAnnotationIPs, + vmi: vmiWithPasst, + }), ) func foregroundDeleteOptions() *client.DeleteOptions {