Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 16 additions & 10 deletions aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import (
"context"
"fmt"
"sort"
"strings"
"time"

"github.com/gruntwork-io/cloud-nuke/util"
"github.com/hashicorp/go-multierror"

commonTelemetry "github.com/gruntwork-io/go-commons/telemetry"

Expand Down Expand Up @@ -157,7 +157,8 @@ func IsNukeable(resourceType string, resourceTypes []string) bool {
return false
}

func nukeAllResourcesInRegion(account *AwsAccountResources, region string, collector *reporting.Collector) {
func nukeAllResourcesInRegion(ctx context.Context, account *AwsAccountResources, region string, collector *reporting.Collector) error {
var allErrors *multierror.Error
resourcesInRegion := account.Resources[region]

for _, awsResource := range resourcesInRegion.Resources {
Expand All @@ -175,7 +176,7 @@ func nukeAllResourcesInRegion(account *AwsAccountResources, region string, colle
BatchSize: len(batch),
})

results, err := (*awsResource).Nuke(context.TODO(), batch)
results, err := (*awsResource).Nuke(ctx, batch)

// Emit ResourceDeleted for each result
for _, result := range results {
Expand All @@ -194,14 +195,16 @@ func nukeAllResourcesInRegion(account *AwsAccountResources, region string, colle

if err != nil {
// Handle rate limiting
if strings.Contains(err.Error(), "RequestLimitExceeded") {
if util.IsThrottlingError(err) {
logging.Debug(
"Request limit reached. Waiting 1 minute before making new requests",
)
time.Sleep(1 * time.Minute)
continue
}

allErrors = multierror.Append(allErrors, fmt.Errorf("[%s] %s: %w", region, (*awsResource).ResourceName(), err))

// Report to telemetry - aggregated metrics of failures per resources.
telemetry.TrackEvent(commonTelemetry.EventContext{
EventName: fmt.Sprintf("error:Nuke:%s", (*awsResource).ResourceName()),
Expand All @@ -216,13 +219,17 @@ func nukeAllResourcesInRegion(account *AwsAccountResources, region string, colle
}
}
}

return allErrors.ErrorOrNil()
}

// NukeAllResources - Nukes all aws resources
func NukeAllResources(account *AwsAccountResources, regions []string, collector *reporting.Collector) error {
func NukeAllResources(ctx context.Context, account *AwsAccountResources, regions []string, collector *reporting.Collector) error {
// Emit NukeStarted event (CLIRenderer will initialize progress bar)
collector.Emit(reporting.NukeStarted{Total: account.TotalResourceCount()})

var allErrors *multierror.Error

telemetry.TrackEvent(commonTelemetry.EventContext{
EventName: "Begin nuking resources",
}, map[string]interface{}{})
Expand All @@ -234,10 +241,9 @@ func NukeAllResources(account *AwsAccountResources, regions []string, collector
"region": region,
})

// We intentionally do not handle an error returned from this method, because we collect individual errors
// on per-resource basis via the collector. In the run report displayed at the end of
// a cloud-nuke run, we show exactly which resources deleted cleanly and which encountered errors
nukeAllResourcesInRegion(account, region, collector)
if err := nukeAllResourcesInRegion(ctx, account, region, collector); err != nil {
allErrors = multierror.Append(allErrors, err)
}
telemetry.TrackEvent(commonTelemetry.EventContext{
EventName: "Done Nuking Region",
}, map[string]interface{}{
Expand All @@ -249,5 +255,5 @@ func NukeAllResources(account *AwsAccountResources, regions []string, collector
// Emit NukeComplete event (triggers final output in renderers)
collector.Emit(reporting.NukeComplete{})

return nil
return allErrors.ErrorOrNil()
}
2 changes: 1 addition & 1 deletion commands/aws_commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ func awsNukeHelper(c *cli.Context, configObj config.Config, query *aws.Query, ou

// Execute the nuke operation if confirmed
if shouldProceed {
return aws.NukeAllResources(account, query.Regions, collector)
return aws.NukeAllResources(c.Context, account, query.Regions, collector)
}

return nil
Expand Down
13 changes: 13 additions & 0 deletions util/error.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,16 @@ type ResourceExecutionTimeout struct {
func (err ResourceExecutionTimeout) Error() string {
return fmt.Sprintf("execution timed out after: %v", err.Timeout)
}

// IsThrottlingError checks if the error is an AWS API throttling error
// using structured error code matching via smithy.APIError.
func IsThrottlingError(err error) bool {
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "RequestLimitExceeded", "ThrottlingException", "TooManyRequestsException":
return true
}
}
return false
}