From 480e13d21b228d2690d3f34b36f445b438e304fe Mon Sep 17 00:00:00 2001 From: Sjuul Janssen Date: Tue, 3 Jan 2023 11:58:25 +0100 Subject: [PATCH] upgraded to aws sdk v2 --- basic/collector.go | 18 ++--- basic/scraper.go | 39 +++++----- enhanced/collector.go | 6 +- enhanced/scraper.go | 109 ++++++++++++++------------- enhanced/scraper_test.go | 12 +-- go.mod | 17 ++++- go.sum | 40 ++++++++-- sessions/sessions.go | 150 +++++++++++++++++++------------------- sessions/sessions_test.go | 6 +- 9 files changed, 214 insertions(+), 183 deletions(-) diff --git a/basic/collector.go b/basic/collector.go index ad4dab26..86190998 100644 --- a/basic/collector.go +++ b/basic/collector.go @@ -29,19 +29,19 @@ type Metric struct { } type Collector struct { - config *config.Config - sessions *sessions.Sessions - metrics []Metric - l log.Logger + config *config.Config + awsConfigs *sessions.Configs + metrics []Metric + l log.Logger } // New creates a new instance of a Collector. -func New(config *config.Config, sessions *sessions.Sessions) *Collector { +func New(config *config.Config, awsConfigs *sessions.Configs) *Collector { return &Collector{ - config: config, - sessions: sessions, - metrics: Metrics, - l: log.With("component", "basic"), + config: config, + awsConfigs: awsConfigs, + metrics: Metrics, + l: log.With("component", "basic"), } } diff --git a/basic/scraper.go b/basic/scraper.go index b4d47718..e3fac926 100644 --- a/basic/scraper.go +++ b/basic/scraper.go @@ -1,11 +1,13 @@ package basic import ( + "context" "sync" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + cloudwatchtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/prometheus/client_golang/prometheus" "github.com/percona/rds_exporter/config" @@ -24,17 +26,17 @@ type Scraper struct { ch chan<- prometheus.Metric // internal - svc *cloudwatch.CloudWatch + svc *cloudwatch.Client constLabels prometheus.Labels } func NewScraper(instance *config.Instance, collector *Collector, ch chan<- prometheus.Metric) *Scraper { // Create CloudWatch client - sess, _ := collector.sessions.GetSession(instance.Region, instance.Instance) - if sess == nil { + awsConfig, _ := collector.awsConfigs.GetSession(instance.Region, instance.Instance) + if awsConfig == nil { return nil } - svc := cloudwatch.New(sess) + svc := cloudwatch.NewFromConfig(*awsConfig) constLabels := prometheus.Labels{ "region": instance.Region, @@ -60,12 +62,12 @@ func NewScraper(instance *config.Instance, collector *Collector, ch chan<- prome } } -func getLatestDatapoint(datapoints []*cloudwatch.Datapoint) *cloudwatch.Datapoint { - var latest *cloudwatch.Datapoint = nil +func getLatestDatapoint(datapoints []cloudwatchtypes.Datapoint) *cloudwatchtypes.Datapoint { + var latest *cloudwatchtypes.Datapoint = nil for dp := range datapoints { if latest == nil || latest.Timestamp.Before(*datapoints[dp].Timestamp) { - latest = datapoints[dp] + latest = &datapoints[dp] } } @@ -94,26 +96,25 @@ func (s *Scraper) Scrape() { func (s *Scraper) scrapeMetric(metric Metric) error { now := time.Now() end := now.Add(-Delay) + period := int32(Period.Seconds()) params := &cloudwatch.GetMetricStatisticsInput{ - EndTime: aws.Time(end), - StartTime: aws.Time(end.Add(-Range)), - - Period: aws.Int64(int64(Period.Seconds())), + EndTime: aws.Time(end), + StartTime: aws.Time(end.Add(-Range)), + Period: &period, MetricName: aws.String(metric.cwName), Namespace: aws.String("AWS/RDS"), - Dimensions: []*cloudwatch.Dimension{}, - Statistics: aws.StringSlice([]string{"Average"}), - Unit: nil, + Dimensions: []cloudwatchtypes.Dimension{}, + Statistics: []cloudwatchtypes.Statistic{"Average"}, } - params.Dimensions = append(params.Dimensions, &cloudwatch.Dimension{ + params.Dimensions = append(params.Dimensions, cloudwatchtypes.Dimension{ Name: aws.String("DBInstanceIdentifier"), Value: aws.String(s.instance.Instance), }) // Call CloudWatch to gather the datapoints - resp, err := s.svc.GetMetricStatistics(params) + resp, err := s.svc.GetMetricStatistics(context.TODO(), params) if err != nil { return err } @@ -127,7 +128,7 @@ func (s *Scraper) scrapeMetric(metric Metric) error { dp := getLatestDatapoint(resp.Datapoints) // Get the metric. - v := aws.Float64Value(dp.Average) + v := *dp.Average switch metric.cwName { case "EngineUptime": // "Fake EngineUptime -> node_boot_time with time.Now().Unix() - EngineUptime." diff --git a/enhanced/collector.go b/enhanced/collector.go index 8f0c902c..771028ff 100644 --- a/enhanced/collector.go +++ b/enhanced/collector.go @@ -13,7 +13,7 @@ import ( // Collector collects enhanced RDS metrics by utilizing several scrapers. type Collector struct { - sessions *sessions.Sessions + sessions *sessions.Configs logger log.Logger rw sync.RWMutex @@ -27,14 +27,14 @@ const ( ) // NewCollector creates new collector and starts scrapers. -func NewCollector(sessions *sessions.Sessions) *Collector { +func NewCollector(sessions *sessions.Configs) *Collector { c := &Collector{ sessions: sessions, logger: log.With("component", "enhanced"), metrics: make(map[string][]prometheus.Metric), } - for session, instances := range sessions.AllSessions() { + for session, instances := range sessions.AllConfigs() { s := newScraper(session, instances) interval := maxInterval diff --git a/enhanced/scraper.go b/enhanced/scraper.go index 3db5abb0..2de633e9 100644 --- a/enhanced/scraper.go +++ b/enhanced/scraper.go @@ -5,9 +5,8 @@ import ( "fmt" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/log" @@ -18,14 +17,14 @@ import ( type scraper struct { instances []sessions.Instance logStreamNames []string - svc *cloudwatchlogs.CloudWatchLogs + svc *cloudwatchlogs.Client nextStartTime time.Time logger log.Logger testDisallowUnknownFields bool // for tests only } -func newScraper(session *session.Session, instances []sessions.Instance) *scraper { +func newScraper(config *aws.Config, instances []sessions.Instance) *scraper { logStreamNames := make([]string, 0, len(instances)) for _, instance := range instances { logStreamNames = append(logStreamNames, instance.ResourceID) @@ -34,7 +33,7 @@ func newScraper(session *session.Session, instances []sessions.Instance) *scrape return &scraper{ instances: instances, logStreamNames: logStreamNames, - svc: cloudwatchlogs.New(session), + svc: cloudwatchlogs.NewFromConfig(*config), nextStartTime: time.Now().Add(-3 * time.Minute).Round(0), // strip monotonic clock reading logger: log.With("component", "enhanced"), } @@ -78,68 +77,66 @@ func (s *scraper) scrape(ctx context.Context) (map[string][]prometheus.Metric, m input := &cloudwatchlogs.FilterLogEventsInput{ LogGroupName: aws.String("RDSOSMetrics"), - LogStreamNames: aws.StringSlice(s.logStreamNames[sliceStart:sliceEnd]), - StartTime: aws.Int64(aws.TimeUnixMilli(s.nextStartTime)), + LogStreamNames: s.logStreamNames[sliceStart:sliceEnd], + StartTime: aws.Int64(s.nextStartTime.UnixMilli()), } s.logger.With("next_start", s.nextStartTime.UTC()).With("since_last", time.Since(s.nextStartTime)).Debugf("Requesting metrics") - // collect all returned events and metrics/messages - collectAllMetrics := func(output *cloudwatchlogs.FilterLogEventsOutput, lastPage bool) bool { - for _, event := range output.Events { - l := s.logger.With("EventId", *event.EventId).With("LogStreamName", *event.LogStreamName) - l = l.With("Timestamp", aws.MillisecondsTimeValue(event.Timestamp).UTC()) - l = l.With("IngestionTime", aws.MillisecondsTimeValue(event.IngestionTime).UTC()) - - var instance *sessions.Instance - for _, i := range s.instances { - if i.ResourceID == *event.LogStreamName { - instance = &i - break - } - } - if instance == nil { - l.Errorf("Failed to find instance.") - continue - } + output, err := s.svc.FilterLogEvents(ctx, input) + if err != nil { + s.logger.Errorf("Failed to filter log events: %s.", err) + } - if instance.DisableEnhancedMetrics { - l.Debugf("Enhanced Metrics are disabled for instance %v.", instance) - continue - } - l = l.With("region", instance.Region).With("instance", instance.Instance) - - // l.Debugf("Message:\n%s", *event.Message) - osMetrics, err := parseOSMetrics([]byte(*event.Message), s.testDisallowUnknownFields) - if err != nil { - // only for tests - if s.testDisallowUnknownFields { - panic(fmt.Sprintf("New metrics should be added: %s", err)) - } - - l.Errorf("Failed to parse metrics: %s.", err) - continue - } - // l.Debugf("OS Metrics:\n%#v", osMetrics) + for _, event := range output.Events { + l := s.logger.With("EventId", *event.EventId).With("LogStreamName", *event.LogStreamName) - timestamp := aws.MillisecondsTimeValue(event.Timestamp).UTC() - l.Debugf("Timestamp from message: %s; from event: %s.", osMetrics.Timestamp.UTC(), timestamp) + l = l.With("Timestamp", time.Unix(*event.Timestamp, 0).UTC()) + l = l.With("IngestionTime", time.Unix(*event.IngestionTime, 0).UTC()) - if allMetrics[instance.ResourceID] == nil { - allMetrics[instance.ResourceID] = make(map[time.Time][]prometheus.Metric) + var instance *sessions.Instance + for _, i := range s.instances { + if i.ResourceID == *event.LogStreamName { + instance = &i + break } - allMetrics[instance.ResourceID][timestamp] = osMetrics.makePrometheusMetrics(instance.Region, instance.Labels) + } + if instance == nil { + l.Errorf("Failed to find instance.") + continue + } - if allMessages[instance.ResourceID] == nil { - allMessages[instance.ResourceID] = make(map[time.Time]string) + if instance.DisableEnhancedMetrics { + l.Debugf("Enhanced Metrics are disabled for instance %v.", instance) + continue + } + l = l.With("region", instance.Region).With("instance", instance.Instance) + + // l.Debugf("Message:\n%s", *event.Message) + osMetrics, err := parseOSMetrics([]byte(*event.Message), s.testDisallowUnknownFields) + if err != nil { + // only for tests + if s.testDisallowUnknownFields { + panic(fmt.Sprintf("New metrics should be added: %s", err)) } - allMessages[instance.ResourceID][timestamp] = *event.Message + + l.Errorf("Failed to parse metrics: %s.", err) + continue } + // l.Debugf("OS Metrics:\n%#v", osMetrics) - return true // continue pagination - } - if err := s.svc.FilterLogEventsPagesWithContext(ctx, input, collectAllMetrics); err != nil { - s.logger.Errorf("Failed to filter log events: %s.", err) + timestamp := time.Unix(*event.Timestamp, 0).UTC() + l.Debugf("Timestamp from message: %s; from event: %s.", osMetrics.Timestamp.UTC(), timestamp) + + if allMetrics[instance.ResourceID] == nil { + allMetrics[instance.ResourceID] = make(map[time.Time][]prometheus.Metric) + } + allMetrics[instance.ResourceID][timestamp] = osMetrics.makePrometheusMetrics(instance.Region, instance.Labels) + + if allMessages[instance.ResourceID] == nil { + allMessages[instance.ResourceID] = make(map[time.Time]string) + } + allMessages[instance.ResourceID][timestamp] = *event.Message } } // get better times diff --git a/enhanced/scraper_test.go b/enhanced/scraper_test.go index 84e0d054..b833f30c 100644 --- a/enhanced/scraper_test.go +++ b/enhanced/scraper_test.go @@ -44,11 +44,11 @@ func TestScraper(t *testing.T) { sess, err := sessions.New(cfg.Instances, client.HTTP(), false) require.NoError(t, err) - for session, instances := range sess.AllSessions() { - session, instances := session, instances + for config, instances := range sess.AllConfigs() { + config, instances := config, instances t.Run(fmt.Sprint(instances), func(t *testing.T) { // test that there are no new metrics - s := newScraper(session, instances) + s := newScraper(config, instances) s.testDisallowUnknownFields = true metrics, messages := s.scrape(context.Background()) require.Len(t, metrics, len(instances)) @@ -159,10 +159,10 @@ func TestScraperDisableEnhancedMetrics(t *testing.T) { return false } - for session, instances := range sess.AllSessions() { - session, instances := session, instances + for config, instances := range sess.AllConfigs() { + config, instances := config, instances t.Run(fmt.Sprint(instances), func(t *testing.T) { - s := newScraper(session, instances) + s := newScraper(config, instances) s.testDisallowUnknownFields = true metrics, _ := s.scrape(context.Background()) diff --git a/go.mod b/go.mod index e6a576ec..de3bfc13 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,13 @@ module github.com/percona/rds_exporter go 1.17 require ( - github.com/aws/aws-sdk-go v1.36.30 + github.com/aws/aws-sdk-go-v2 v1.17.3 + github.com/aws/aws-sdk-go-v2/config v1.18.7 + github.com/aws/aws-sdk-go-v2/credentials v1.13.7 + github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.23.1 + github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.17.3 + github.com/aws/aws-sdk-go-v2/service/rds v1.38.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 github.com/percona/exporter_shared v0.7.3 github.com/prometheus/client_golang v1.10.0 github.com/prometheus/common v0.24.0 @@ -15,6 +21,14 @@ require ( require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -28,7 +42,6 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect diff --git a/go.sum b/go.sum index 16693b75..e2cb312a 100644 --- a/go.sum +++ b/go.sum @@ -22,9 +22,37 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.36.30 h1:hAwyfe7eZa7sM+S5mIJZFiNFwJMia9Whz6CYblioLoU= -github.com/aws/aws-sdk-go v1.36.30/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= +github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E= +github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.23.1 h1:6VwY6q6RZwxZTTTXjDmS8qbeBKvWwp8ugMKCEBjdgWA= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.23.1/go.mod h1:th8fks2kW4FFCUKUQenuEG9TEzMLVxeL0ckdJn/QVbI= +github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.17.3 h1:GKDlULxx6rUH67l/CRnG0xZzeMLZVk5gVCkVqNK6bgg= +github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.17.3/go.mod h1:xHK1ta0bQEa5jL6rahKRJvsibjzDO7NTIs5itzsF4w8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= +github.com/aws/aws-sdk-go-v2/service/rds v1.38.0 h1:4LyiHO7LPBNQmP8QbK4pa/9wgjpaI16Z4PGZVNJhKTA= +github.com/aws/aws-sdk-go-v2/service/rds v1.38.0/go.mod h1:Ume9NHqT871hUdxIRojWtWsPFyCswQmSjHHhyGot7v0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -99,8 +127,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -331,7 +360,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -362,14 +390,12 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -388,8 +414,6 @@ golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapK golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/sessions/sessions.go b/sessions/sessions.go index 8c321b85..58e976f9 100644 --- a/sessions/sessions.go +++ b/sessions/sessions.go @@ -1,17 +1,21 @@ +//https://aws.github.io/aws-sdk-go-v2/docs/migrating/ + package sessions import ( + "context" "fmt" "net/http" "os" "text/tabwriter" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/rds" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/prometheus/common/log" "github.com/percona/rds_exporter/config" @@ -37,24 +41,24 @@ func (i Instance) String() string { return res } -// Sessions is a pool of AWS sessions. -type Sessions struct { - sessions map[*session.Session][]Instance +// Configs is a pool of AWS configs. +type Configs struct { + configs map[*aws.Config][]Instance } // New creates a new sessions pool for given configuration. -func New(instances []config.Instance, client *http.Client, trace bool) (*Sessions, error) { +func New(instances []config.Instance, client *http.Client, trace bool) (*Configs, error) { logger := log.With("component", "sessions") logger.Info("Creating sessions...") - res := &Sessions{ - sessions: make(map[*session.Session][]Instance), + res := &Configs{ + configs: make(map[*aws.Config][]Instance), } - sharedSessions := make(map[string]*session.Session) // region/key => session + configs := make(map[string]*aws.Config) // region/key => session for _, instance := range instances { // re-use session for the same region and key (explicit or empty for implicit) pair - if s := sharedSessions[instance.Region+"/"+instance.AWSAccessKey]; s != nil { - res.sessions[s] = append(res.sessions[s], Instance{ + if s := configs[instance.Region+"/"+instance.AWSAccessKey]; s != nil { + res.configs[s] = append(res.configs[s], Instance{ Region: instance.Region, Instance: instance.Instance, Labels: instance.Labels, @@ -64,41 +68,14 @@ func New(instances []config.Instance, client *http.Client, trace bool) (*Session continue } - // use given credentials, or default credential chain - var creds *credentials.Credentials - - creds, err := buildCredentials(instance) + awsCfg, err := buildConfig(instance, client, trace) if err != nil { return nil, err } - // make config with careful logging - awsCfg := &aws.Config{ - Credentials: creds, - Region: aws.String(instance.Region), - HTTPClient: client, - } - if trace { - // fail-safe - if _, ok := os.LookupEnv("CI"); ok { - panic("Do not enable AWS request tracing on CI - output will contain credentials.") - } - - awsCfg.Logger = aws.LoggerFunc(logger.Debug) - awsCfg.CredentialsChainVerboseErrors = aws.Bool(true) - level := aws.LogDebugWithSigning | aws.LogDebugWithHTTPBody - level |= aws.LogDebugWithRequestRetries | aws.LogDebugWithRequestErrors | aws.LogDebugWithEventStreamBody - awsCfg.LogLevel = aws.LogLevel(level) - } - - // store session - s, err := session.NewSession(awsCfg) - if err != nil { - return nil, err - } - sharedSessions[instance.Region+"/"+instance.AWSAccessKey] = s - res.sessions[s] = append(res.sessions[s], Instance{ + configs[instance.Region+"/"+instance.AWSAccessKey] = awsCfg + res.configs[awsCfg] = append(res.configs[awsCfg], Instance{ Region: instance.Region, Instance: instance.Instance, Labels: instance.Labels, @@ -108,11 +85,11 @@ func New(instances []config.Instance, client *http.Client, trace bool) (*Session } // add resource ID to all instances - for session, instances := range res.sessions { - svc := rds.New(session) + for config, instances := range res.configs { + svc := rds.NewFromConfig(*config) var marker *string for { - output, err := svc.DescribeDBInstances(&rds.DescribeDBInstancesInput{ + output, err := svc.DescribeDBInstances(context.TODO(), &rds.DescribeDBInstancesInput{ Marker: marker, }) if err != nil { @@ -135,7 +112,7 @@ func New(instances []config.Instance, client *http.Client, trace bool) (*Session } // remove instances without resource ID - for session, instances := range res.sessions { + for session, instances := range res.configs { newInstances := make([]Instance, 0, len(instances)) for _, instance := range instances { if instance.ResourceID == "" { @@ -144,65 +121,84 @@ func New(instances []config.Instance, client *http.Client, trace bool) (*Session } newInstances = append(newInstances, instance) } - res.sessions[session] = newInstances + res.configs[session] = newInstances } // remove sessions without instances - for _, s := range sharedSessions { - if len(res.sessions[s]) == 0 { - delete(res.sessions, s) + for _, s := range configs { + if len(res.configs[s]) == 0 { + delete(res.configs, s) } } w := tabwriter.NewWriter(os.Stderr, 0, 0, 2, ' ', 0) fmt.Fprintf(w, "Region\tInstance\tResource ID\tInterval\n") - for _, instances := range res.sessions { + for _, instances := range res.configs { for _, instance := range instances { fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", instance.Region, instance.Instance, instance.ResourceID, instance.EnhancedMonitoringInterval) } } _ = w.Flush() - logger.Infof("Using %d sessions.", len(res.sessions)) + logger.Infof("Using %d sessions.", len(res.configs)) return res, nil } // GetSession returns session and full instance information for given region and instance. -func (s *Sessions) GetSession(region, instance string) (*session.Session, *Instance) { - for session, instances := range s.sessions { +func (s *Configs) GetSession(region, instance string) (*aws.Config, *Instance) { + for config, instances := range s.configs { for _, i := range instances { if i.Region == region && i.Instance == instance { - return session, &i + return config, &i } } } return nil, nil } -func buildCredentials(instance config.Instance) (*credentials.Credentials, error) { - if instance.AWSRoleArn != "" { - stsSession, err := session.NewSession(&aws.Config{ - Region: aws.String(instance.Region), - Credentials: credentials.NewStaticCredentials(instance.AWSAccessKey, instance.AWSSecretKey, ""), - }) - if err != nil { - return nil, err +func buildConfig(instance config.Instance, httpClient *http.Client, trace bool) (*aws.Config, error) { + ctx := context.TODO() + + options := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(instance.Region), awsconfig.WithHTTPClient(httpClient), + } + + if trace { + // fail-safe + if _, ok := os.LookupEnv("CI"); ok { + panic("Do not enable AWS request tracing on CI - output will contain credentials.") } + level := aws.LogSigning | aws.LogRequestWithBody + level |= aws.LogRetries | aws.LogRequest | aws.LogRequestWithBody | aws.LogResponseWithBody + options = append(options, awsconfig.WithClientLogMode(level)) + } + + if instance.AWSAccessKey != "" && instance.AWSSecretKey != "" { + creds := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(instance.AWSAccessKey, instance.AWSSecretKey, "")) + options = append(options, awsconfig.WithCredentialsProvider(creds)) + } + + cfg, err := awsconfig.LoadDefaultConfig(ctx, options...) - return stscreds.NewCredentials(stsSession, instance.AWSRoleArn), nil + if err != nil { + return nil, err } - if instance.AWSAccessKey != "" || instance.AWSSecretKey != "" { - return credentials.NewCredentials(&credentials.StaticProvider{ - Value: credentials.Value{ - AccessKeyID: instance.AWSAccessKey, - SecretAccessKey: instance.AWSSecretKey, - }, - }), nil + + if instance.AWSRoleArn != "" { + client := sts.NewFromConfig(cfg) + creds := stscreds.NewAssumeRoleProvider(client, instance.AWSRoleArn) + return &aws.Config{ + Credentials: creds, + Region: instance.Region, + HTTPClient: httpClient, + ClientLogMode: cfg.ClientLogMode, + }, nil } - return nil, nil + + return &cfg, nil } -// AllSessions returns all sessions and instances. -func (s *Sessions) AllSessions() map[*session.Session][]Instance { - return s.sessions +// AllConfigs returns all aws configs and instances. +func (s *Configs) AllConfigs() map[*aws.Config][]Instance { + return s.configs } diff --git a/sessions/sessions_test.go b/sessions/sessions_test.go index bd459b4b..d280cac0 100644 --- a/sessions/sessions_test.go +++ b/sessions/sessions_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -84,8 +84,8 @@ func TestSession(t *testing.T) { assert.Equal(t, &ap11iExpected, ap11i) assert.Nil(t, ni) - all := sessions.AllSessions() - assert.Equal(t, map[*session.Session][]Instance{ + all := sessions.AllConfigs() + assert.Equal(t, map[*aws.Config][]Instance{ am56s: {am56iExpected}, p10s: {p10iExpected}, m57s: {m57iExpected, ap11iExpected},