Skip to content

Commit 0620f60

Browse files
bennervbennerv
committed
test/e2e: detach NSGs from subnets before deletion
1 parent 5df745b commit 0620f60

File tree

4 files changed

+114
-4
lines changed

4 files changed

+114
-4
lines changed

‎test/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ require (
99
github.com/Azure/ARO-Tools v0.0.0-20251217212052-baa2e4c6d49a
1010
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2
1111
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0
12+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0
1213
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
1314
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0
1415
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0

‎test/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsI
9494
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38=
9595
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0 h1:akP6VpxJGgQRpDR1P462piz/8OhYLRCreDj48AyNabc=
9696
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.2.0/go.mod h1:8wzvopPfyZYPaQUoKW87Zfdul7jmJMDfp/k7YY3oJyA=
97+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 h1:HYGD75g0bQ3VO/Omedm54v4LrD3B1cGImuRF3AJ5wLo=
98+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0/go.mod h1:ulHyBFJOI0ONiRL4vcJTmS7rx18jQQlEPmAgo80cRdM=
9799
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeploymentstacks v1.0.1 h1:bcgO/crpp7wqI0Froi/I4C2fme7Vk/WLusbV399Do8I=
98100
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armdeploymentstacks v1.0.1/go.mod h1:kvfPmsE8gpOwwC1qrO1FeyBDDNfnwBN5UU3MPNiWW7I=
99101
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armfeatures v1.2.0 h1:wIDqH4WA5uJ6irRqjzodeSw6Pmp0tu3oIbwzBZEdMfQ=

‎test/util/framework/per_test_framework.go‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939

4040
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
4141
armcompute "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
42+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
4243
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
4344
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions"
4445

@@ -58,6 +59,7 @@ type perItOrDescribeTestContext struct {
5859
armComputeClientFactory *armcompute.ClientFactory
5960
armResourcesClientFactory *armresources.ClientFactory
6061
armSubscriptionsClientFactory *armsubscriptions.ClientFactory
62+
armNetworkClientFactory *armnetwork.ClientFactory
6163
graphClient *graphutil.Client
6264

6365
timingMetadata timing.SpecTimingMetadata
@@ -349,6 +351,11 @@ func (tc *perItOrDescribeTestContext) cleanupResourceGroup(ctx context.Context,
349351
return err
350352
}
351353

354+
networkClientFactory, err := tc.GetARMNetworkClientFactory(ctx)
355+
if err != nil {
356+
return err
357+
}
358+
352359
hcpClientFactory, err := tc.Get20240610ClientFactory(ctx)
353360
if err != nil {
354361
return err
@@ -371,7 +378,7 @@ func (tc *perItOrDescribeTestContext) cleanupResourceGroup(ctx context.Context,
371378
}
372379

373380
ginkgo.GinkgoLogr.Info("deleting resource group", "resourceGroup", resourceGroupName)
374-
if err := DeleteResourceGroup(ctx, resourceClientFactory.NewResourceGroupsClient(), resourceGroupName, false, timeout); err != nil {
381+
if err := DeleteResourceGroup(ctx, resourceClientFactory.NewResourceGroupsClient(), networkClientFactory, resourceGroupName, false, timeout); err != nil {
375382
return fmt.Errorf("failed to cleanup resource group: %w", err)
376383
}
377384

@@ -398,14 +405,19 @@ func (tc *perItOrDescribeTestContext) cleanupResourceGroupNoRP(ctx context.Conte
398405
return fmt.Errorf("failed to search for managed resource groups: %w", err)
399406
}
400407

401-
clientFactory, err := tc.GetARMResourcesClientFactory(ctx)
408+
resourceClientFactory, err := tc.GetARMResourcesClientFactory(ctx)
409+
if err != nil {
410+
return err
411+
}
412+
413+
networkClientFactory, err := tc.GetARMNetworkClientFactory(ctx)
402414
if err != nil {
403415
return err
404416
}
405417

406418
for _, managedRG := range managedResourceGroups {
407419
ginkgo.GinkgoLogr.Info("deleting managed resource group", "resourceGroup", managedRG, "parentResourceGroup", resourceGroupName)
408-
if err := DeleteResourceGroup(ctx, clientFactory.NewResourceGroupsClient(), managedRG, true, timeout); err != nil {
420+
if err := DeleteResourceGroup(ctx, resourceClientFactory.NewResourceGroupsClient(), networkClientFactory, managedRG, true, timeout); err != nil {
409421
if isIgnorableResourceGroupCleanupError(err) {
410422
ginkgo.GinkgoLogr.Info("ignoring not found resource group", "resourceGroup", managedRG)
411423
} else {
@@ -415,7 +427,7 @@ func (tc *perItOrDescribeTestContext) cleanupResourceGroupNoRP(ctx context.Conte
415427
}
416428

417429
ginkgo.GinkgoLogr.Info("deleting resource group", "resourceGroup", resourceGroupName)
418-
if err := DeleteResourceGroup(ctx, clientFactory.NewResourceGroupsClient(), resourceGroupName, false, timeout); err != nil {
430+
if err := DeleteResourceGroup(ctx, resourceClientFactory.NewResourceGroupsClient(), networkClientFactory, resourceGroupName, false, timeout); err != nil {
419431
return fmt.Errorf("failed to cleanup resource group: %w", err)
420432
}
421433

@@ -564,6 +576,44 @@ func (tc *perItOrDescribeTestContext) getARMSubscriptionsClientFactoryUnlocked()
564576
return tc.armSubscriptionsClientFactory, nil
565577
}
566578

579+
func (tc *perItOrDescribeTestContext) GetARMNetworkClientFactory(ctx context.Context) (*armnetwork.ClientFactory, error) {
580+
tc.contextLock.RLock()
581+
if tc.armNetworkClientFactory != nil {
582+
defer tc.contextLock.RUnlock()
583+
return tc.armNetworkClientFactory, nil
584+
}
585+
tc.contextLock.RUnlock()
586+
587+
tc.contextLock.Lock()
588+
defer tc.contextLock.Unlock()
589+
590+
return tc.getARMNetworkClientFactoryUnlocked(ctx)
591+
}
592+
593+
func (tc *perItOrDescribeTestContext) getARMNetworkClientFactoryUnlocked(ctx context.Context) (*armnetwork.ClientFactory, error) {
594+
if tc.armNetworkClientFactory != nil {
595+
return tc.armNetworkClientFactory, nil
596+
}
597+
598+
creds, err := tc.perBinaryInvocationTestContext.getAzureCredentials()
599+
if err != nil {
600+
return nil, err
601+
}
602+
603+
// We already hold the lock, so we call the unlocked version
604+
subscriptionID, err := tc.getSubscriptionIDUnlocked(ctx)
605+
if err != nil {
606+
return nil, err
607+
}
608+
609+
clientFactory, err := armnetwork.NewClientFactory(subscriptionID, creds, tc.perBinaryInvocationTestContext.getClientFactoryOptions())
610+
if err != nil {
611+
return nil, err
612+
}
613+
tc.armNetworkClientFactory = clientFactory
614+
return tc.armNetworkClientFactory, nil
615+
}
616+
567617
func (tc *perItOrDescribeTestContext) GetARMResourcesClientFactory(ctx context.Context) (*armresources.ClientFactory, error) {
568618
tc.contextLock.RLock()
569619
if tc.armResourcesClientFactory != nil {

‎test/util/framework/resourcegroup_helper.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626

2727
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
2828
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
29+
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
2930
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
3031
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions"
3132
)
@@ -122,6 +123,7 @@ func ListAllExpiredResourceGroups(
122123
func DeleteResourceGroup(
123124
ctx context.Context,
124125
resourceGroupsClient *armresources.ResourceGroupsClient,
126+
networkClientFactory *armnetwork.ClientFactory,
125127
resourceGroupName string,
126128
force bool,
127129
timeout time.Duration,
@@ -130,6 +132,12 @@ func DeleteResourceGroup(
130132
ctx, cancel := context.WithTimeoutCause(ctx, timeout, fmt.Errorf("timeout '%f' minutes exceeded during DeleteResourceGroup for resource group %s", timeout.Minutes(), resourceGroupName))
131133
defer cancel()
132134

135+
// detach any NSGs from subnets to avoid blocking deletion of the RG
136+
err := detachSubnetNSGs(ctx, networkClientFactory, resourceGroupName)
137+
if err != nil {
138+
return fmt.Errorf("failed to detach NSGs from subnets in resource group %s: %w", resourceGroupName, err)
139+
}
140+
133141
var opts *armresources.ResourceGroupsClientBeginDeleteOptions
134142
if force {
135143
opts = &armresources.ResourceGroupsClientBeginDeleteOptions{
@@ -164,3 +172,52 @@ func DeleteResourceGroup(
164172

165173
return nil
166174
}
175+
176+
// detach any NSGs from subnets to avoid blocking deletion of the RG
177+
func detachSubnetNSGs(ctx context.Context, networkClientFactory *armnetwork.ClientFactory, resourceGroupName string) error {
178+
vnetClient := networkClientFactory.NewVirtualNetworksClient()
179+
subnetClient := networkClientFactory.NewSubnetsClient()
180+
181+
vnetPager := vnetClient.NewListPager(resourceGroupName, nil)
182+
for vnetPager.More() {
183+
vnetPage, err := vnetPager.NextPage(ctx)
184+
if err != nil {
185+
return fmt.Errorf("failed listing vnets in resource group %s: %w", resourceGroupName, err)
186+
}
187+
188+
for _, vnet := range vnetPage.Value {
189+
if vnet == nil || vnet.Properties == nil || vnet.Properties.Subnets == nil || vnet.Name == nil {
190+
continue
191+
}
192+
193+
subnetsPager := subnetClient.NewListPager(resourceGroupName, *vnet.Name, nil)
194+
for subnetsPager.More() {
195+
subnetPage, err := subnetsPager.NextPage(ctx)
196+
if err != nil {
197+
return fmt.Errorf("failed listing subnets in resource group %s: %w", resourceGroupName, err)
198+
}
199+
200+
for _, subnet := range subnetPage.Value {
201+
if subnet == nil || subnet.Name == nil || subnet.Properties == nil || subnet.Properties.NetworkSecurityGroup == nil || subnet.Properties.NetworkSecurityGroup.ID == nil {
202+
continue
203+
}
204+
205+
subnet.Properties.NetworkSecurityGroup = nil
206+
poller, err := subnetClient.BeginCreateOrUpdate(ctx, resourceGroupName, *vnet.Name, *subnet.Name, *subnet, &armnetwork.SubnetsClientBeginCreateOrUpdateOptions{})
207+
if err != nil {
208+
return fmt.Errorf("failed detaching NSG from subnet %s in resource group %s: %w", *subnet.Name, resourceGroupName, err)
209+
}
210+
211+
_, err = poller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{
212+
Frequency: StandardPollInterval,
213+
})
214+
if err != nil {
215+
return fmt.Errorf("failed waiting for subnet %s in resource group %s to finish updating: %w", *subnet.Name, resourceGroupName, err)
216+
}
217+
}
218+
}
219+
}
220+
}
221+
222+
return nil
223+
}

0 commit comments

Comments
 (0)