Added integration test for snapshots metrics

Author: psmitj

README.md

@@ -145,6 +145,13 @@ To disable exporting cluster settings use:
 prometheus.cluster.settings: false
 ```
 
+#### Snapshot metrics
+
+To enable exporting snapshot metrics use:
+```
+prometheus.snapshots: true
+```
+
 #### Nodes filter
 
 Metrics include statistics about individual OpenSearch nodes.

build.gradle

@@ -1,3 +1,4 @@
+import org.opensearch.gradle.PropertyNormalization
 import org.opensearch.gradle.test.RestIntegTestTask
 
 import java.util.regex.Matcher
@@ -124,8 +125,13 @@ tasks.named("check").configure { dependsOn(integTest) }
 // Temporary disable task :testingConventions
 testingConventions.enabled = false
 
+// Directory for snapshot repository
+File repositoryDir = new File(project.buildDir, "shared-repository")
+
 testClusters.all {
     numberOfNodes = 2
+    // Configuring repo path for 'fs' type snapshot repository
+    setting 'path.repo', "${repositoryDir.absolutePath}", PropertyNormalization.IGNORE_VALUE
 
     // It seems cluster name can not be customized here. It gives an error:
     // Testclusters does not allow the following settings to be changed:[cluster.name] for node{::yamlRestTest-0}

src/main/java/org/compuscene/metrics/prometheus/PrometheusSettings.java

@@ -55,23 +55,23 @@ public enum INDEX_FILTER_OPTIONS {
     static String PROMETHEUS_SELECTED_OPTION_KEY = "prometheus.indices_filter.selected_option";
 
     /**
-     * This setting is used configure weather to expose cluster settings metrics or not. The default value is true.
+     * This setting is used configure whether to expose cluster settings metrics or not. The default value is true.
      * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_CLUSTER_SETTINGS_KEY}.
      */
     public static final Setting<Boolean> PROMETHEUS_CLUSTER_SETTINGS =
             Setting.boolSetting(PROMETHEUS_CLUSTER_SETTINGS_KEY, true,
                     Setting.Property.Dynamic, Setting.Property.NodeScope);
 
     /**
-     * This setting is used configure weather to expose low level index metrics or not. The default value is true.
+     * This setting is used configure whether to expose low level index metrics or not. The default value is true.
      * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_INDICES_KEY}.
      */
     public static final Setting<Boolean> PROMETHEUS_INDICES =
             Setting.boolSetting(PROMETHEUS_INDICES_KEY, true,
                     Setting.Property.Dynamic, Setting.Property.NodeScope);
 
     /**
-     * This setting is used configure weather to expose snapshot metrics or not. The default value is false.
+     * This setting is used configure whether to expose snapshot metrics or not. The default value is false.
      * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_SNAPSHOTS_KEY}.
      */
     public static final Setting<Boolean> PROMETHEUS_SNAPSHOTS =

src/yamlRestTest/resources/rest-api-spec/test/70_10_snapshots_metrics.yml

@@ -0,0 +1,50 @@
+---
+"Verify snapshots metrics enabled":
+
+  # Create 'fs' snapshot repository
+  - do:
+      snapshot.create_repository:
+        repository: test_repo_get_1
+        body:
+          type: fs
+          settings:
+            location: "test_repo_get_1_loc"
+
+  - do:
+      snapshot.get_repository: { }
+
+  - is_true: test_repo_get_1
+
+  # Enable snapshots metrics
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            prometheus.snapshots: "true"
+        flat_settings: true
+
+  - match: { persistent: { prometheus.snapshots: "true" } }
+
+  # Create snapshot
+  - do:
+      snapshot.create:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_1
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: test_snapshot_1 }
+  - match: { snapshot.state : SUCCESS }
+
+  # Fetch and verify metrics
+  - do:
+      prometheus.metrics: {}
+
+  - match:
+      $body: |
+        /.*
+        \# \s* HELP \s+ opensearch_min_snapshot_age \s+.*\n
+        \# \s* TYPE \s+ opensearch_min_snapshot_age \s+ gauge\n
+        opensearch_min_snapshot_age\{
+          cluster="yamlRestTest",sm_policy="adhoc",
+        \}\s+\d+\.\d+\n
+        .*/