[vSphere][datastore_cluster] Add support for new metrics in datastore_cluster metricset (#40634)

kush-elastic · web-flow · commit 30320a3bfcfc · 2024-09-02T10:33:59.000+05:30
* initial commit for datastore cluster

* mage update

* update:
- data.json
- field.yml

* doc updates

* datastore_cluster -&gt; datastorecluster

* resolve review comments

* fix UTs

* update URL for UTs

* update UTs

* fix field mapping

* update data.json
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -67134,6 +67134,47 @@ type: long
 
 --
 
+[float]
+=== datastorecluster
+
+Datastore Cluster
+
+
+
+*`vsphere.datastorecluster.name`*::
++
+--
+The Datastore Cluster name.
+
+
+type: keyword
+
+--
+
+*`vsphere.datastorecluster.capacity.bytes`*::
++
+--
+Total capacity of this storage pod, in bytes.
+
+
+type: long
+
+format: bytes
+
+--
+
+*`vsphere.datastorecluster.free_space.bytes`*::
++
+--
+Total free space on this storage pod, in bytes.
+
+
+type: long
+
+format: bytes
+
+--
+
 [float]
 === host
 
diff --git a/metricbeat/docs/modules/vsphere.asciidoc b/metricbeat/docs/modules/vsphere.asciidoc
@@ -11,7 +11,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any Vmware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0 and 6.5.
 
-By default it enables the metricsets `cluster`, `network`, `resourcepool`, `datastore`, `host` and `virtualmachine`.
+By default it enables the metricsets `cluster`, `datastore`, `datastorecluster`, `host`, `network`, `resourcepool` and `virtualmachine`.
 
 [float]
 === Dashboard
@@ -35,7 +35,7 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: vsphere
   enabled: true
-  metricsets: ["cluster", "datastore", "host", "virtualmachine", "network", "resourcepool"]
+  metricsets: ["cluster", "datastore", "datastorecluster", "host", "network", "resourcepool", "virtualmachine"]
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]
@@ -57,6 +57,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-vsphere-datastore,datastore>>
 
+* <<metricbeat-metricset-vsphere-datastorecluster,datastorecluster>>
+
 * <<metricbeat-metricset-vsphere-host,host>>
 
 * <<metricbeat-metricset-vsphere-network,network>>
@@ -69,6 +71,8 @@ include::vsphere/cluster.asciidoc[]
 
 include::vsphere/datastore.asciidoc[]
 
+include::vsphere/datastorecluster.asciidoc[]
+
 include::vsphere/host.asciidoc[]
 
 include::vsphere/network.asciidoc[]
diff --git a/metricbeat/docs/modules/vsphere/datastorecluster.asciidoc b/metricbeat/docs/modules/vsphere/datastorecluster.asciidoc
@@ -0,0 +1,29 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/vsphere/datastorecluster/_meta/docs.asciidoc
+
+
+[[metricbeat-metricset-vsphere-datastorecluster]]
+=== vSphere datastorecluster metricset
+
+beta[]
+
+include::../../../module/vsphere/datastorecluster/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+:edit_url:
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-vsphere,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/vsphere/datastorecluster/_meta/data.json[]
+----
+:edit_url!:
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -311,8 +311,9 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-module-uwsgi,uWSGI>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-uwsgi-status,status>>   
 |<<metricbeat-module-vsphere,vSphere>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.6+| .6+|  |<<metricbeat-metricset-vsphere-cluster,cluster>> beta[]  
+.7+| .7+|  |<<metricbeat-metricset-vsphere-cluster,cluster>> beta[]  
 |<<metricbeat-metricset-vsphere-datastore,datastore>>   
+|<<metricbeat-metricset-vsphere-datastorecluster,datastorecluster>> beta[]  
 |<<metricbeat-metricset-vsphere-host,host>>   
 |<<metricbeat-metricset-vsphere-network,network>> beta[]  
 |<<metricbeat-metricset-vsphere-resourcepool,resourcepool>> beta[]  
diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -1007,7 +1007,7 @@ metricbeat.modules:
 #------------------------------- VSphere Module -------------------------------
 - module: vsphere
   enabled: true
-  metricsets: ["cluster", "datastore", "host", "virtualmachine", "network", "resourcepool"]
+  metricsets: ["cluster", "datastore", "datastorecluster", "host", "network", "resourcepool", "virtualmachine"]
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]
diff --git a/metricbeat/module/vsphere/_meta/README.md b/metricbeat/module/vsphere/_meta/README.md
@@ -1,7 +1,7 @@
 # Testing using GOVCSIM.
 
 
-To test the vsphere module without a real Vmware SDK URL you can use the following setup. Govcsim is a vCenter Server and ESXi API based simulator written using govmomi. It creates a vCenter Server model with a datacenter, hosts, cluster, resource pools, networks and a datastore.
+To test the vsphere module without a real Vmware SDK URL you can use the following setup. Govcsim is a vCenter Server and ESXi API based simulator written using govmomi. It creates a vCenter Server model with a datacenter, datastore cluster, hosts, cluster, resource pools, networks and a datastore.
 
 
 Requirements:
@@ -30,10 +30,11 @@ Now setup your metricbeat config to connect to Govcsim:
   metricsets:
     - cluster
     - datastore
+    - datastorecluster
     - host
-    - virtualmachine
     - network
     - resourcepool
+    - virtualmachine
   enabled: true
   period: 5s
   hosts: ["https://127.0.0.1:8989/sdk"]
diff --git a/metricbeat/module/vsphere/_meta/config.reference.yml b/metricbeat/module/vsphere/_meta/config.reference.yml
@@ -1,6 +1,6 @@
 - module: vsphere
   enabled: true
-  metricsets: ["cluster", "datastore", "host", "virtualmachine", "network", "resourcepool"]
+  metricsets: ["cluster", "datastore", "datastorecluster", "host", "network", "resourcepool", "virtualmachine"]
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]
diff --git a/metricbeat/module/vsphere/_meta/config.yml b/metricbeat/module/vsphere/_meta/config.yml
@@ -1,11 +1,12 @@
 - module: vsphere
-  #metricsets:
+  # metricsets:
   #  - cluster
   #  - datastore
+  #  - datastorecluster
   #  - host
-  #  - virtualmachine
   #  - network
   #  - resourcepool
+  #  - virtualmachine
   # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds.
   period: 20s
   hosts: ["https://localhost/sdk"]
diff --git a/metricbeat/module/vsphere/_meta/docs.asciidoc b/metricbeat/module/vsphere/_meta/docs.asciidoc
@@ -1,6 +1,6 @@
 The vSphere module uses the https://github.com/vmware/govmomi[Govmomi] library to collect metrics from any Vmware SDK URL (ESXi/VCenter). This library is built for and tested against ESXi and vCenter 5.5, 6.0 and 6.5.
 
-By default it enables the metricsets `cluster`, `network`, `resourcepool`, `datastore`, `host` and `virtualmachine`.
+By default it enables the metricsets `cluster`, `datastore`, `datastorecluster`, `host`, `network`, `resourcepool` and `virtualmachine`.
 
 [float]
 === Dashboard
diff --git a/metricbeat/module/vsphere/datastorecluster/_meta/data.json b/metricbeat/module/vsphere/datastorecluster/_meta/data.json
@@ -0,0 +1,27 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "service": {
+        "address": "https://localhost:8980/sdk",
+        "type": "vsphere"
+    },
+    "event": {
+        "dataset": "vsphere.datastorecluster",
+        "module": "vsphere",
+        "duration": 15443161
+    },
+    "metricset": {
+        "period": 20000,
+        "name": "datastorecluster"
+    },
+    "vsphere": {
+        "datastorecluster": {
+            "name": "datastore_cluster1",
+            "capacity": {
+                "bytes": 8795019280384
+            },
+            "free_space": {
+                "bytes": 8788836876288
+            }
+        }
+    }
+}
diff --git a/metricbeat/module/vsphere/datastorecluster/_meta/docs.asciidoc b/metricbeat/module/vsphere/datastorecluster/_meta/docs.asciidoc
@@ -0,0 +1 @@
+This is the Datastore Cluster metricset of the module vsphere.
diff --git a/metricbeat/module/vsphere/datastorecluster/_meta/fields.yml b/metricbeat/module/vsphere/datastorecluster/_meta/fields.yml
@@ -0,0 +1,20 @@
+- name: datastorecluster
+  type: group
+  release: beta
+  description: >
+    Datastore Cluster
+  fields:
+    - name: name
+      type: keyword
+      description: >
+        The Datastore Cluster name.
+    - name: capacity.bytes
+      type: long
+      description: >
+        Total capacity of this storage pod, in bytes.
+      format: bytes
+    - name: free_space.bytes
+      type: long
+      description: >
+        Total free space on this storage pod, in bytes.
+      format: bytes
diff --git a/metricbeat/module/vsphere/datastorecluster/data.go b/metricbeat/module/vsphere/datastorecluster/data.go
@@ -0,0 +1,34 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package datastorecluster
+
+import (
+	"github.com/vmware/govmomi/vim25/mo"
+
+	"github.com/elastic/elastic-agent-libs/mapstr"
+)
+
+func (m *DatastoreClusterMetricSet) mapEvent(datastoreCluster mo.StoragePod) mapstr.M {
+	event := mapstr.M{}
+
+	event.Put("name", datastoreCluster.Name)
+	event.Put("capacity.bytes", datastoreCluster.Summary.Capacity)
+	event.Put("free_space.bytes", datastoreCluster.Summary.FreeSpace)
+
+	return event
+}
diff --git a/metricbeat/module/vsphere/datastorecluster/data_test.go b/metricbeat/module/vsphere/datastorecluster/data_test.go
@@ -0,0 +1,51 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package datastorecluster
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/types"
+)
+
+func TestEventMapping(t *testing.T) {
+	datastoreClusterTest := mo.StoragePod{
+		Summary: &types.StoragePodSummary{
+			Capacity:  100,
+			FreeSpace: 50,
+		},
+		Folder: mo.Folder{
+			ManagedEntity: mo.ManagedEntity{
+				Name: "Folder1",
+			},
+		},
+	}
+
+	event := (&DatastoreClusterMetricSet{}).mapEvent(datastoreClusterTest)
+
+	name, _ := event.GetValue("name")
+	assert.Equal(t, "Folder1", name)
+
+	capacity, _ := event.GetValue("capacity.bytes")
+	assert.Equal(t, int64(100), capacity)
+
+	freeSpace, _ := event.GetValue("free_space.bytes")
+	assert.Equal(t, int64(50), freeSpace)
+}
diff --git a/metricbeat/module/vsphere/datastorecluster/datastorecluster.go b/metricbeat/module/vsphere/datastorecluster/datastorecluster.go
diff --git a/metricbeat/module/vsphere/datastorecluster/datastorecluster_test.go b/metricbeat/module/vsphere/datastorecluster/datastorecluster_test.go
diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go
diff --git a/metricbeat/modules.d/vsphere.yml.disabled b/metricbeat/modules.d/vsphere.yml.disabled
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
