diff --git a/aws/aws.go b/aws/aws.go index 9120ed83..83f478cf 100644 --- a/aws/aws.go +++ b/aws/aws.go @@ -4,10 +4,10 @@ import ( "context" "fmt" "sort" - "strings" "time" "github.com/gruntwork-io/cloud-nuke/util" + "github.com/hashicorp/go-multierror" commonTelemetry "github.com/gruntwork-io/go-commons/telemetry" @@ -157,7 +157,8 @@ func IsNukeable(resourceType string, resourceTypes []string) bool { return false } -func nukeAllResourcesInRegion(account *AwsAccountResources, region string, collector *reporting.Collector) { +func nukeAllResourcesInRegion(ctx context.Context, account *AwsAccountResources, region string, collector *reporting.Collector) error { + var allErrors *multierror.Error resourcesInRegion := account.Resources[region] for _, awsResource := range resourcesInRegion.Resources { @@ -175,7 +176,7 @@ func nukeAllResourcesInRegion(account *AwsAccountResources, region string, colle BatchSize: len(batch), }) - results, err := (*awsResource).Nuke(context.TODO(), batch) + results, err := (*awsResource).Nuke(ctx, batch) // Emit ResourceDeleted for each result for _, result := range results { @@ -194,7 +195,7 @@ func nukeAllResourcesInRegion(account *AwsAccountResources, region string, colle if err != nil { // Handle rate limiting - if strings.Contains(err.Error(), "RequestLimitExceeded") { + if util.IsThrottlingError(err) { logging.Debug( "Request limit reached. Waiting 1 minute before making new requests", ) @@ -202,6 +203,8 @@ func nukeAllResourcesInRegion(account *AwsAccountResources, region string, colle continue } + allErrors = multierror.Append(allErrors, fmt.Errorf("[%s] %s: %w", region, (*awsResource).ResourceName(), err)) + // Report to telemetry - aggregated metrics of failures per resources. telemetry.TrackEvent(commonTelemetry.EventContext{ EventName: fmt.Sprintf("error:Nuke:%s", (*awsResource).ResourceName()), @@ -216,13 +219,17 @@ func nukeAllResourcesInRegion(account *AwsAccountResources, region string, colle } } } + + return allErrors.ErrorOrNil() } // NukeAllResources - Nukes all aws resources -func NukeAllResources(account *AwsAccountResources, regions []string, collector *reporting.Collector) error { +func NukeAllResources(ctx context.Context, account *AwsAccountResources, regions []string, collector *reporting.Collector) error { // Emit NukeStarted event (CLIRenderer will initialize progress bar) collector.Emit(reporting.NukeStarted{Total: account.TotalResourceCount()}) + var allErrors *multierror.Error + telemetry.TrackEvent(commonTelemetry.EventContext{ EventName: "Begin nuking resources", }, map[string]interface{}{}) @@ -234,10 +241,9 @@ func NukeAllResources(account *AwsAccountResources, regions []string, collector "region": region, }) - // We intentionally do not handle an error returned from this method, because we collect individual errors - // on per-resource basis via the collector. In the run report displayed at the end of - // a cloud-nuke run, we show exactly which resources deleted cleanly and which encountered errors - nukeAllResourcesInRegion(account, region, collector) + if err := nukeAllResourcesInRegion(ctx, account, region, collector); err != nil { + allErrors = multierror.Append(allErrors, err) + } telemetry.TrackEvent(commonTelemetry.EventContext{ EventName: "Done Nuking Region", }, map[string]interface{}{ @@ -249,5 +255,5 @@ func NukeAllResources(account *AwsAccountResources, regions []string, collector // Emit NukeComplete event (triggers final output in renderers) collector.Emit(reporting.NukeComplete{}) - return nil + return allErrors.ErrorOrNil() } diff --git a/commands/aws_commands.go b/commands/aws_commands.go index 33c90a41..c751dd09 100644 --- a/commands/aws_commands.go +++ b/commands/aws_commands.go @@ -150,7 +150,7 @@ func awsNukeHelper(c *cli.Context, configObj config.Config, query *aws.Query, ou // Execute the nuke operation if confirmed if shouldProceed { - return aws.NukeAllResources(account, query.Regions, collector) + return aws.NukeAllResources(c.Context, account, query.Regions, collector) } return nil diff --git a/util/error.go b/util/error.go index 9b1897c9..3e87ce8c 100644 --- a/util/error.go +++ b/util/error.go @@ -62,3 +62,16 @@ type ResourceExecutionTimeout struct { func (err ResourceExecutionTimeout) Error() string { return fmt.Sprintf("execution timed out after: %v", err.Timeout) } + +// IsThrottlingError checks if the error is an AWS API throttling error +// using structured error code matching via smithy.APIError. +func IsThrottlingError(err error) bool { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "RequestLimitExceeded", "ThrottlingException", "TooManyRequestsException": + return true + } + } + return false +}