[8.x](backport #40461) x-pack/metricbeat/module/gcp: Add Organization ID and display name to cloud labels

CHANGELOG.asciidoc

@@ -3,6 +3,16 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.15.2]]
+=== Beats version 8.15.2
+https://github.com/elastic/beats/compare/v8.15.0\...v8.15.2[View commits]
+
+==== Breaking changes
+
+*Metricbeat*
+
+- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461]
+
 [[release-notes-8.15.1]]
 === Beats version 8.15.1
 https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits]

CHANGELOG.next.asciidoc

@@ -63,6 +63,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added back `elasticsearch.node.stats.jvm.mem.pools.*` to the `node_stats` metricset {pull}40571[40571]
 - Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683]
 - Add support for specifying a custom endpoint for GCP service clients. {issue}40848[40848] {pull}40918[40918]
+- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461]
 
 *Osquerybeat*
 

metricbeat/docs/modules/gcp.asciidoc

@@ -136,6 +136,10 @@ Generally, you have to create a Service Account and assign it the following role
 - `compute.instances.get`
 - `compute.instances.list`
 
+* `Browser`:
+- `resourcemanager.projects.get`
+- `resourcemanager.organizations.get`
+
 You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above.
 
 [float]
@@ -145,6 +149,45 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics
 
 If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls).
 
+We have updated our field names to align with ECS semantics. As part of this change:
+
+*  `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID).
+*  `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name).
+*  New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively.
+
+To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration:
+[source,json]
+----
+{
+  "processors": [
+    {
+      "set": {
+        "field": "cloud.account.id",
+        "value": "{{cloud.project.id}}",
+        "if": "ctx?.cloud?.project?.id != null"
+      }
+    },
+    {
+      "set": {
+        "field": "cloud.account.name",
+        "value": "{{cloud.project.name}}",
+        "if": "ctx?.cloud?.project?.name != null"
+      }
+    },
+    {
+      "remove": {
+        "field": [
+          "cloud.project.id",
+          "cloud.project.name"
+        ],
+        "ignore_missing": true
+      }
+    }
+  ]
+}
+----
+For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide.
+
 If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation.
 GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details.
 In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API.

x-pack/metricbeat/module/gcp/_meta/docs.asciidoc

@@ -124,6 +124,10 @@ Generally, you have to create a Service Account and assign it the following role
 - `compute.instances.get`
 - `compute.instances.list`
 
+* `Browser`:
+- `resourcemanager.projects.get`
+- `resourcemanager.organizations.get`
+
 You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above.
 
 [float]
@@ -133,6 +137,45 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics
 
 If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls).
 
+We have updated our field names to align with ECS semantics. As part of this change:
+
+*  `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID).
+*  `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name).
+*  New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively.
+
+To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration:
+[source,json]
+----
+{
+  "processors": [
+    {
+      "set": {
+        "field": "cloud.account.id",
+        "value": "{{cloud.project.id}}",
+        "if": "ctx?.cloud?.project?.id != null"
+      }
+    },
+    {
+      "set": {
+        "field": "cloud.account.name",
+        "value": "{{cloud.project.name}}",
+        "if": "ctx?.cloud?.project?.name != null"
+      }
+    },
+    {
+      "remove": {
+        "field": [
+          "cloud.project.id",
+          "cloud.project.name"
+        ],
+        "ignore_missing": true
+      }
+    }
+  ]
+}
+----
+For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide.
+
 If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation.
 GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details.
 In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API.

x-pack/metricbeat/module/gcp/constants.go

@@ -48,9 +48,9 @@ const (
 
 	ECSCloudRegion = "region"
 
-	ECSCloudAccount     = "account"
-	ECSCloudAccountID   = "id"
-	ECSCloudAccountName = "name"
+	ECSCloudAccount = "account"
+	ECSCloudID      = "id"
+	ECSCloudName    = "name"
 
 	ECSCloudInstance        = "instance"
 	ECSCloudInstanceKey     = ECSCloud + "." + ECSCloudInstance
@@ -63,6 +63,7 @@ const (
 	ECSCloudMachineKey     = ECSCloud + "." + ECSCloudMachine
 	ECSCloudMachineType    = "type"
 	ECSCloudMachineTypeKey = ECSCloudMachineKey + "." + ECSCloudMachineType
+	ECSCloudProject        = "project"
 )
 
 // Metadata keys used for events. They follow GCP structure.

x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go

@@ -19,15 +19,18 @@ import (
 )
 
 // NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource.
-func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) {
+func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) {
 	return &metadataCollector{
-		projectID: projectID,
-		zone:      zone,
-		region:    region,
-		regions:   regions,
-		opt:       opt,
-		instances: make(map[string]*sqladmin.DatabaseInstance),
-		logger:    logp.NewLogger("metrics-cloudsql"),
+		projectID:        projectID,
+		projectName:      projectName,
+		organizationID:   organizationID,
+		organizationName: organizationName,
+		zone:             zone,
+		region:           region,
+		regions:          regions,
+		opt:              opt,
+		instances:        make(map[string]*sqladmin.DatabaseInstance),
+		logger:           logp.NewLogger("metrics-cloudsql"),
 	}, nil
 }
 
@@ -46,11 +49,14 @@ type cloudsqlMetadata struct {
 }
 
 type metadataCollector struct {
-	projectID string
-	zone      string
-	region    string
-	regions   []string
-	opt       []option.ClientOption
+	projectID        string
+	projectName      string
+	organizationID   string
+	organizationName string
+	zone             string
+	region           string
+	regions          []string
+	opt              []option.ClientOption
 	// NOTE: instances holds data used for all metrics collected in a given period
 	// this avoids calling the remote endpoint for each metric, which would take a long time overall
 	instances map[string]*sqladmin.DatabaseInstance
@@ -91,7 +97,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim
 		return gcp.MetadataCollectorData{}, err
 	}
 
-	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp)
+	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName)
 
 	metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp)
 	if err != nil {

x-pack/metricbeat/module/gcp/metrics/compute/metadata.go

@@ -23,9 +23,12 @@ import (
 )
 
 // NewMetadataService returns the specific Metadata service for a GCP Compute resource
-func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) {
+func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) {
 	return &metadataCollector{
 		projectID:        projectID,
+		projectName:      projectName,
+		organizationID:   organizationID,
+		organizationName: organizationName,
 		zone:             zone,
 		region:           region,
 		regions:          regions,
@@ -50,6 +53,9 @@ type computeMetadata struct {
 
 type metadataCollector struct {
 	projectID        string
+	projectName      string
+	organizationID   string
+	organizationName string
 	zone             string
 	region           string
 	regions          []string
@@ -64,7 +70,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim
 	if err != nil {
 		return gcp.MetadataCollectorData{}, err
 	}
-	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp)
+	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName)
 	metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp)
 	if err != nil {
 		return gcp.MetadataCollectorData{}, err

x-pack/metricbeat/module/gcp/metrics/metadata_services.go

@@ -16,11 +16,11 @@ import (
 func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) {
 	switch serviceName {
 	case gcp.ServiceCompute:
-		return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...)
+		return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...)
 	case gcp.ServiceCloudSQL:
-		return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...)
+		return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...)
 	case gcp.ServiceRedis:
-		return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...)
+		return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...)
 	default:
 		return nil, nil
 	}

x-pack/metricbeat/module/gcp/metrics/metricset.go

@@ -18,6 +18,8 @@ import (
 	"google.golang.org/genproto/googleapis/api/metric"
 	"google.golang.org/protobuf/types/known/durationpb"
 
+	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
+
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp"
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -107,8 +109,11 @@ type config struct {
 	CredentialsJSON     string   `config:"credentials_json"`
 	Endpoint            string   `config:"endpoint"`
 
-	opt    []option.ClientOption
-	period *durationpb.Duration
+	opt              []option.ClientOption
+	period           *durationpb.Duration
+	organizationID   string
+	organizationName string
+	projectName      string
 }
 
 // New creates a new instance of the MetricSet. New is responsible for unpacking
@@ -158,6 +163,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	// Get ingest delay and sample period for each metric type
 	ctx := context.Background()
+	// set organization id
+	if errs := m.setOrgAndProjectDetails(ctx); errs != nil {
+		m.Logger().Warnf("error occurred while fetching organization and project details: %s", errs)
+	}
 	client, err := monitoring.NewMetricClient(ctx, m.config.opt...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating Stackdriver client: %w", err)
@@ -358,3 +367,61 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M {
 	}
 	return hostRootFields
 }
+
+func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) []error {
+	var errs []error
+
+	// Initialize the Cloud Resource Manager service
+	srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...)
+	if err != nil {
+		errs = append(errs, fmt.Errorf("failed to create cloudresourcemanager service: %w", err))
+		return errs
+	}
+	// Set Project name
+	err = m.setProjectDetails(ctx, srv)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	//Set Organization Details
+	err = m.setOrganizationDetails(ctx, srv)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	return errs
+}
+
+func (m *MetricSet) setProjectDetails(ctx context.Context, service *cloudresourcemanager.Service) error {
+	project, err := service.Projects.Get(m.config.ProjectID).Context(ctx).Do()
+	if err != nil {
+		return fmt.Errorf("failed to get project name: %w", err)
+	}
+	if project != nil {
+		m.config.projectName = project.Name
+	}
+	return nil
+}
+
+func (m *MetricSet) setOrganizationDetails(ctx context.Context, service *cloudresourcemanager.Service) error {
+	// Get the project ancestor details
+	ancestryResponse, err := service.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do()
+	if err != nil {
+		return fmt.Errorf("failed to get project ancestors: %w", err)
+	}
+	if len(ancestryResponse.Ancestor) == 0 {
+		return fmt.Errorf("no ancestors found for project '%s'", m.config.ProjectID)
+	}
+	ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1]
+
+	if ancestor.ResourceId.Type == "organization" {
+		m.config.organizationID = ancestor.ResourceId.Id
+		orgReq := service.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID))
+
+		orgDetails, err := orgReq.Context(ctx).Do()
+		if err != nil {
+			return fmt.Errorf("failed to get organization details: %w", err)
+		}
+
+		m.config.organizationName = orgDetails.DisplayName
+	}
+	return nil
+}