uptake kineticpulse library for metrics logic in acdc (#51)

Author: schowsf

acdc-ws/app/Filters.scala

@@ -1,9 +1,9 @@
-import utils.MetricFilter
-
 import javax.inject.Inject
 
 import play.api.http.DefaultHttpFilters
 
+import com.salesforce.mce.kineticpulse.MetricFilter
+
 class Filters @Inject() (
   metricsFilter: MetricFilter
 ) extends DefaultHttpFilters(metricsFilter)

acdc-ws/app/Module.scala

@@ -7,7 +7,8 @@
 
 import com.google.inject.AbstractModule
 
-import services.{Metric, PrometheusMetric}
+import com.salesforce.mce.kineticpulse.{Metric, PrometheusMetric}
+
 import tasks.{AuthSettingReloadTask, DataCountTask, DataInstExpirationTask}
 import utils.{Authorization, AuthorizationSettings}
 

acdc-ws/app/controllers/MetricController.scala

@@ -14,14 +14,13 @@ import com.salesforce.mce.acdc.db.{
   DatasetLineageQuery,
   DatasetQuery
 }
-import services.{DatabaseService, Metric}
-import utils.DbConfig
+import services.DatabaseService
+import utils.{AcdcMetric, DbConfig}
 
 class DataCountTask @Inject() (
   actorSystem: ActorSystem,
   dbService: DatabaseService,
-  dbConfig: DbConfig,
-  metric: Metric
+  dbConfig: DbConfig
 )(implicit
   ec: ExecutionContext
 ) extends Logging {
@@ -39,9 +38,9 @@ class DataCountTask @Inject() (
       logger.debug(s"Counted $datasetInstanceCount records from dataset_instance ...")
       logger.debug(s"Counted $datasetLineageCount records from dataset_lineage ...")
 
-      metric.countDB("dataset", datasetCount.toDouble)
-      metric.countDB("dataset-instance", datasetInstanceCount.toDouble)
-      metric.countDB("dataset-lineage", datasetLineageCount.toDouble)
+      AcdcMetric.countDB("dataset", datasetCount.toDouble)
+      AcdcMetric.countDB("dataset-instance", datasetInstanceCount.toDouble)
+      AcdcMetric.countDB("dataset-lineage", datasetLineageCount.toDouble)
 
       refresh()
     }

acdc-ws/app/services/Metric.scala

@@ -0,0 +1,17 @@
+package utils
+
+import io.prometheus.client.Gauge
+
+object AcdcMetric {
+  val dbCountGauge: Gauge = Gauge
+    .build()
+    .name("acdc_db_record_count")
+    .labelNames("table")
+    .help("acdc DB table records count")
+    .register
+
+  def countDB(table: String, cnt: Double): Unit = {
+    dbCountGauge.labels(table).set(cnt)
+  }
+
+}

acdc-ws/app/tasks/DataCountTask.scala

@@ -39,6 +39,26 @@ acdc.db {
     count-task-frequency-minute = ${?COUNT_TASK_FREQUENCY_MINUTE}
 }
 
+com.salesforce.mce.kineticpulse.metrics {
+
+    enabled = true
+
+    # some http request paths are not interesting to capture metrics such as http duration
+    # Disable metric collection for these request paths
+    bypass.paths = ["/__status", "/", "/__metrics"]
+
+    # routes segments to track in api_duration_seconds_summary, matched values go to the metric label "path"
+    # example:  ["eci", "model/predict", "model/accuracy"]
+    routes-to-track = ["instance", "instances", "dataset", "datasets", "lineage"]
+
+    # delimiter is used within metric label "path" to replace the "/" in the request.path
+    delimiter = "-"
+
+    # label value to be used when a path is not bypass, but did not match routes-to-track,
+    unmatched-path = ""
+
+}
+
 ## Akka
 # https://www.playframework.com/documentation/latest/ScalaAkka#Configuration
 # https://www.playframework.com/documentation/latest/JavaAkka#Configuration
@@ -369,11 +389,3 @@ play.assets {
   path = "/public"
   urlPrefix = "/assets"
 }
-
-# some http request paths are not interesting to capture metrics such as http duration
-acdc.metrics {
-  enable = true
-
-  # Disable metric collection for these request paths
-  bypass.paths = ["/__status", "/", "/__metrics"]
-}

acdc-ws/app/utils/AcdcMetric.scala

@@ -1,4 +1,4 @@
 GET     /__status                   controllers.StatusController.status()
-GET     /__metrics                  controllers.MetricController.collect()
+GET     /__metrics                  com.salesforce.mce.kineticpulse.MetricController.collect()
 
 ->      /api/v1                     routers.ApiRouter

acdc-ws/app/utils/MetricFilter.scala

@@ -1,9 +1,7 @@
 val slickVersion = "3.4.1"
 
 val scalaTestArtifact = "org.scalatest" %% "scalatest" % "3.2.+" % Test
-val prometheusClient = "io.prometheus" % "simpleclient" % "0.16.0"
-val prometheusCommon = "io.prometheus" % "simpleclient_common" % "0.16.0"
-val prometheusHotSpot = "io.prometheus" % "simpleclient_hotspot" % "0.16.0"
+val kineticpulse = "com.salesforce.mce" %% "kineticpulse-metric" % "0.2.+"
 
 lazy val commonSettings = Seq(
   scalacOptions ++= Seq("-deprecation", "-feature", "-Xlint"), // , "-Xfatal-warnings"),
@@ -48,9 +46,7 @@ lazy val ws = (project in file("acdc-ws")).
     buildInfoPackage := "com.salesforce.mce.acdc.ws",
     libraryDependencies ++= Seq(
       guice,
-      prometheusClient,
-      prometheusCommon,
-      prometheusHotSpot
+      kineticpulse
     ),
     dependencyOverrides ++= Seq(
       // fix https://nvd.nist.gov/vuln/detail/CVE-2022-42003