diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6f75297dea0..ffc779cdb70 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,16 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.17.0]] +=== Beats version 8.17.0 +https://github.com/elastic/beats/compare/v8.17.0\...v8.17.0[View commits] + +==== Breaking changes + +*Metricbeat* + +- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] + [[release-notes-8.15.3]] === Beats version 8.15.3 https://github.com/elastic/beats/compare/v8.15.2\...v8.15.3[View commits] @@ -65,7 +75,6 @@ https://github.com/elastic/beats/compare/v8.15.0\...v8.15.2[View commits] - Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] - [[release-notes-8.15.1]] === Beats version 8.15.1 https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8c9836fc42f..3676e209c19 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -67,6 +67,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Added back `elasticsearch.node.stats.jvm.mem.pools.*` to the `node_stats` metricset {pull}40571[40571] - Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683] - Add support for specifying a custom endpoint for GCP service clients. {issue}40848[40848] {pull}40918[40918] +- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] *Osquerybeat* diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index 025b70a9404..00c1536e5c0 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -136,6 +136,10 @@ Generally, you have to create a Service Account and assign it the following role - `compute.instances.get` - `compute.instances.list` +* `Browser`: +- `resourcemanager.projects.get` +- `resourcemanager.organizations.get` + You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above. [float] @@ -145,6 +149,45 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). +We have updated our field names to align with ECS semantics. As part of this change: + +* `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID). +* `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). +* New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. + +To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration: +[source,json] +---- +{ + "processors": [ + { + "set": { + "field": "cloud.account.id", + "value": "{{cloud.project.id}}", + "if": "ctx?.cloud?.project?.id != null" + } + }, + { + "set": { + "field": "cloud.account.name", + "value": "{{cloud.project.name}}", + "if": "ctx?.cloud?.project?.name != null" + } + }, + { + "remove": { + "field": [ + "cloud.project.id", + "cloud.project.name" + ], + "ignore_missing": true + } + } + ] +} +---- +For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. diff --git a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc index 27a326fe78d..80b4ff08358 100644 --- a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc @@ -124,6 +124,10 @@ Generally, you have to create a Service Account and assign it the following role - `compute.instances.get` - `compute.instances.list` +* `Browser`: +- `resourcemanager.projects.get` +- `resourcemanager.organizations.get` + You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above. [float] @@ -133,6 +137,45 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). +We have updated our field names to align with ECS semantics. As part of this change: + +* `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID). +* `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). +* New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. + +To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration: +[source,json] +---- +{ + "processors": [ + { + "set": { + "field": "cloud.account.id", + "value": "{{cloud.project.id}}", + "if": "ctx?.cloud?.project?.id != null" + } + }, + { + "set": { + "field": "cloud.account.name", + "value": "{{cloud.project.name}}", + "if": "ctx?.cloud?.project?.name != null" + } + }, + { + "remove": { + "field": [ + "cloud.project.id", + "cloud.project.name" + ], + "ignore_missing": true + } + } + ] +} +---- +For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index 787cb63f56a..bba1171bb58 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -48,9 +48,9 @@ const ( ECSCloudRegion = "region" - ECSCloudAccount = "account" - ECSCloudAccountID = "id" - ECSCloudAccountName = "name" + ECSCloudAccount = "account" + ECSCloudID = "id" + ECSCloudName = "name" ECSCloudInstance = "instance" ECSCloudInstanceKey = ECSCloud + "." + ECSCloudInstance @@ -63,6 +63,7 @@ const ( ECSCloudMachineKey = ECSCloud + "." + ECSCloudMachine ECSCloudMachineType = "type" ECSCloudMachineTypeKey = ECSCloudMachineKey + "." + ECSCloudMachineType + ECSCloudProject = "project" ) // Metadata keys used for events. They follow GCP structure. diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index 0c21127d81f..6598af23ee6 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -19,15 +19,18 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource. -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instances: make(map[string]*sqladmin.DatabaseInstance), - logger: logp.NewLogger("metrics-cloudsql"), + projectID: projectID, + projectName: projectName, + organizationID: organizationID, + organizationName: organizationName, + zone: zone, + region: region, + regions: regions, + opt: opt, + instances: make(map[string]*sqladmin.DatabaseInstance), + logger: logp.NewLogger("metrics-cloudsql"), }, nil } @@ -46,11 +49,14 @@ type cloudsqlMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption + projectID string + projectName string + organizationID string + organizationName string + zone string + region string + regions []string + opt []option.ClientOption // NOTE: instances holds data used for all metrics collected in a given period // this avoids calling the remote endpoint for each metric, which would take a long time overall instances map[string]*sqladmin.DatabaseInstance @@ -91,7 +97,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go index eb9f5971303..2036a7861ce 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go @@ -23,9 +23,12 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Compute resource -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ projectID: projectID, + projectName: projectName, + organizationID: organizationID, + organizationName: organizationName, zone: zone, region: region, regions: regions, @@ -50,6 +53,9 @@ type computeMetadata struct { type metadataCollector struct { projectID string + projectName string + organizationID string + organizationName string zone string region string regions []string @@ -64,7 +70,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim if err != nil { return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { return gcp.MetadataCollectorData{}, err diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index 3722157eab9..95c7971915a 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -16,11 +16,11 @@ import ( func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) { switch serviceName { case gcp.ServiceCompute: - return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) case gcp.ServiceCloudSQL: - return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) case gcp.ServiceRedis: - return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) default: return nil, nil } diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index 274a6153b54..f5b15d68fb3 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -18,6 +18,8 @@ import ( "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/protobuf/types/known/durationpb" + cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" @@ -107,8 +109,11 @@ type config struct { CredentialsJSON string `config:"credentials_json"` Endpoint string `config:"endpoint"` - opt []option.ClientOption - period *durationpb.Duration + opt []option.ClientOption + period *durationpb.Duration + organizationID string + organizationName string + projectName string } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -158,6 +163,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get ingest delay and sample period for each metric type ctx := context.Background() + // set organization id + if errs := m.setOrgAndProjectDetails(ctx); errs != nil { + m.Logger().Warnf("error occurred while fetching organization and project details: %s", errs) + } client, err := monitoring.NewMetricClient(ctx, m.config.opt...) if err != nil { return nil, fmt.Errorf("error creating Stackdriver client: %w", err) @@ -358,3 +367,61 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M { } return hostRootFields } + +func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) []error { + var errs []error + + // Initialize the Cloud Resource Manager service + srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) + if err != nil { + errs = append(errs, fmt.Errorf("failed to create cloudresourcemanager service: %w", err)) + return errs + } + // Set Project name + err = m.setProjectDetails(ctx, srv) + if err != nil { + errs = append(errs, err) + } + //Set Organization Details + err = m.setOrganizationDetails(ctx, srv) + if err != nil { + errs = append(errs, err) + } + return errs +} + +func (m *MetricSet) setProjectDetails(ctx context.Context, service *cloudresourcemanager.Service) error { + project, err := service.Projects.Get(m.config.ProjectID).Context(ctx).Do() + if err != nil { + return fmt.Errorf("failed to get project name: %w", err) + } + if project != nil { + m.config.projectName = project.Name + } + return nil +} + +func (m *MetricSet) setOrganizationDetails(ctx context.Context, service *cloudresourcemanager.Service) error { + // Get the project ancestor details + ancestryResponse, err := service.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do() + if err != nil { + return fmt.Errorf("failed to get project ancestors: %w", err) + } + if len(ancestryResponse.Ancestor) == 0 { + return fmt.Errorf("no ancestors found for project '%s'", m.config.ProjectID) + } + ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1] + + if ancestor.ResourceId.Type == "organization" { + m.config.organizationID = ancestor.ResourceId.Id + orgReq := service.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID)) + + orgDetails, err := orgReq.Context(ctx).Do() + if err != nil { + return fmt.Errorf("failed to get organization details: %w", err) + } + + m.config.organizationName = orgDetails.DisplayName + } + return nil +} diff --git a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go index 1a553cc543f..705b86ec838 100644 --- a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go @@ -21,15 +21,18 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Redis resource -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instances: make(map[string]*redispb.Instance), - logger: logp.NewLogger("metrics-redis"), + projectID: projectID, + projectName: projectName, + organizationID: organizationID, + organizationName: organizationName, + zone: zone, + region: region, + regions: regions, + opt: opt, + instances: make(map[string]*redispb.Instance), + logger: logp.NewLogger("metrics-redis"), }, nil } @@ -48,11 +51,14 @@ type redisMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption + projectID string + projectName string + organizationID string + organizationName string + zone string + region string + regions []string + opt []option.ClientOption // NOTE: instances holds data used for all metrics collected in a given period // this avoids calling the remote endpoint for each metric, which would take a long time overall instances map[string]*redispb.Instance @@ -66,7 +72,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/timeseries.go b/x-pack/metricbeat/module/gcp/metrics/timeseries.go index 0118df39538..0a0d3115450 100644 --- a/x-pack/metricbeat/module/gcp/metrics/timeseries.go +++ b/x-pack/metricbeat/module/gcp/metrics/timeseries.go @@ -116,7 +116,7 @@ func (m *MetricSet) groupTimeSeries(ctx context.Context, timeSeries []timeSeries aligner := tsa.aligner for _, ts := range tsa.timeSeries { if defaultMetadataService == nil { - metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts) + metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.organizationID, m.config.organizationName, m.config.projectName) } sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region, m.config.Regions) keyValues := mapper.mapTimeSeriesToKeyValuesPoints(ts, aligner) diff --git a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go index 028f3d713e6..87f60663e51 100644 --- a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go @@ -27,16 +27,22 @@ func NewStackdriverCollectorInputData(ts *monitoringpb.TimeSeries, projectID, zo // NewStackdriverMetadataServiceForTimeSeries apart from having a long name takes a time series object to return the // Stackdriver canonical Metadata extractor -func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries) MetadataService { +func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries, organizationID, organizationName, projectName string) MetadataService { return &StackdriverTimeSeriesMetadataCollector{ - timeSeries: ts, + timeSeries: ts, + organizationID: organizationID, + organizationName: organizationName, + projectName: projectName, } } // StackdriverTimeSeriesMetadataCollector is the implementation of MetadataCollector to collect metrics from Stackdriver // common TimeSeries objects type StackdriverTimeSeriesMetadataCollector struct { - timeSeries *monitoringpb.TimeSeries + timeSeries *monitoringpb.TimeSeries + organizationID string + organizationName string + projectName string } // Metadata parses a Timeseries object to return its metadata divided into "unknown" (first object) and ECS (second @@ -53,14 +59,19 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i ecs := mapstr.M{ ECSCloud: mapstr.M{ - ECSCloudAccount: mapstr.M{ - ECSCloudAccountID: accountID, - ECSCloudAccountName: accountID, + ECSCloudProject: mapstr.M{ + ECSCloudID: accountID, + ECSCloudName: s.projectName, }, ECSCloudProvider: "gcp", }, } - + if s.organizationID != "" { + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudID, s.organizationID) + } + if s.organizationName != "" { + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudName, s.organizationName) + } if availabilityZone != "" { _, _ = ecs.Put(ECSCloud+"."+ECSCloudAvailabilityZone, availabilityZone)