initial prometheus integration for metrics and health

contrib/docker/docker-compose.prometheus.yml

@@ -0,0 +1,11 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-lifecycle'
+    ports:
+      - "9090:9090"
+    network_mode: host

contrib/docker/prometheus.yml

@@ -0,0 +1,16 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'emissary-health'
+    metrics_path: /api/health
+    params:
+      format: [ "prometheus" ]
+    static_configs:
+      - targets: [ 'localhost:8001' ]
+  - job_name: 'emissary-metrics'
+    metrics_path: /api/metrics
+    params:
+      format: ["prometheus"]
+    static_configs:
+      - targets: ['localhost:8001']

pom.xml

@@ -83,8 +83,10 @@
     <dep.logstash-logback-encoder.version>7.2</dep.logstash-logback-encoder.version>
     <dep.mockito.version>5.17.0</dep.mockito.version>
     <dep.netty.version>4.1.125.Final</dep.netty.version>
+    <dep.opentelemetry.version>1.53.0</dep.opentelemetry.version>
     <dep.opentest4j.version>1.3.0</dep.opentest4j.version>
     <dep.picocli.version>4.7.4</dep.picocli.version>
+    <dep.prometheus.version>0.16.0</dep.prometheus.version>
     <dep.proto-google-common-protos.version>2.26.0</dep.proto-google-common-protos.version>
     <dep.protobuf.version>3.25.5</dep.protobuf.version>
     <dep.resilience4j.version>1.7.0</dep.resilience4j.version>
@@ -229,6 +231,41 @@
         <artifactId>resilience4j-retry</artifactId>
         <version>${dep.resilience4j.version}</version>
       </dependency>
+      <dependency>
+        <groupId>io.opentelemetry</groupId>
+        <artifactId>opentelemetry-api</artifactId>
+        <version>${dep.opentelemetry.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.opentelemetry</groupId>
+        <artifactId>opentelemetry-exporter-prometheus</artifactId>
+        <version>${dep.opentelemetry.version}-alpha</version>
+      </dependency>
+      <dependency>
+        <groupId>io.opentelemetry</groupId>
+        <artifactId>opentelemetry-sdk</artifactId>
+        <version>${dep.opentelemetry.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.opentelemetry</groupId>
+        <artifactId>opentelemetry-sdk-metrics</artifactId>
+        <version>${dep.opentelemetry.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.prometheus</groupId>
+        <artifactId>simpleclient</artifactId>
+        <version>${dep.prometheus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.prometheus</groupId>
+        <artifactId>simpleclient_common</artifactId>
+        <version>${dep.prometheus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.prometheus</groupId>
+        <artifactId>simpleclient_dropwizard</artifactId>
+        <version>${dep.prometheus.version}</version>
+      </dependency>
       <dependency>
         <groupId>jakarta.annotation</groupId>
         <artifactId>jakarta.annotation-api</artifactId>
@@ -403,6 +440,13 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <dependency>
+        <groupId>io.opentelemetry</groupId>
+        <artifactId>opentelemetry-bom</artifactId>
+        <version>${dep.opentelemetry.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-bom</artifactId>
@@ -521,6 +565,21 @@
       <groupId>io.grpc</groupId>
       <artifactId>grpc-stub</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient</artifactId>
+      <version>0.16.0</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_common</artifactId>
+      <version>0.16.0</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_dropwizard</artifactId>
+      <version>0.16.0</version>
+    </dependency>
     <dependency>
       <groupId>jakarta.annotation</groupId>
       <artifactId>jakarta.annotation-api</artifactId>

src/main/java/emissary/core/MetricsManager.java

@@ -16,6 +16,8 @@
 import com.codahale.metrics.health.HealthCheckRegistry;
 import com.codahale.metrics.health.jvm.ThreadDeadlockHealthCheck;
 import com.codahale.metrics.jmx.JmxReporter;
+import com.codahale.metrics.jvm.BufferPoolMetricSet;
+import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
 import com.codahale.metrics.jvm.FileDescriptorRatioGauge;
 import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
@@ -24,12 +26,15 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
 import java.net.InetSocketAddress;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 
+import static java.lang.management.ManagementFactory.getPlatformMBeanServer;
+
 /**
  * Manages the interactions with CodaHale's Metrics package, including configuration
  */
@@ -127,6 +132,32 @@ protected void initMetrics() {
             this.metrics.registerAll(new GarbageCollectorMetricSet());
             this.metrics.registerAll(new ThreadStatesGaugeSet());
             this.metrics.register("file.descriptor.info", new FileDescriptorRatioGauge());
+
+            // Add additional JVM metrics for comprehensive monitoring if enabled
+            if (this.conf.findBooleanEntry("JVM_ADDITIONAL_METRICS_ENABLED", false)) {
+                logger.debug("Additional JVM Metrics are enabled");
+                this.metrics.registerAll(new ClassLoadingGaugeSet());
+                this.metrics.registerAll(new BufferPoolMetricSet(getPlatformMBeanServer()));
+
+                // Add JVM uptime metric
+                this.metrics.register("jvm.uptime", new Gauge<Long>() {
+                    @Override
+                    public Long getValue() {
+                        return ManagementFactory.getRuntimeMXBean().getUptime();
+                    }
+                });
+
+                // Add available processors metric
+                this.metrics.register("jvm.processors", new Gauge<Integer>() {
+                    @Override
+                    public Integer getValue() {
+                        return Runtime.getRuntime().availableProcessors();
+                    }
+                });
+            } else {
+                logger.debug("Additional JVM Metrics are disabled");
+            }
+
         } else {
             logger.debug("JVM Metrics are disabled");
         }

src/main/java/emissary/grpc/retry/RetryHandler.java

@@ -1,9 +1,13 @@
 package emissary.grpc.retry;
 
 import emissary.config.Configurator;
+import emissary.core.MetricsManager;
+import emissary.core.NamespaceException;
 import emissary.grpc.exceptions.PoolException;
 import emissary.grpc.exceptions.ServiceNotAvailableException;
 
+import com.codahale.metrics.Counter;
+import com.google.common.base.VerifyException;
 import io.github.resilience4j.core.IntervalFunction;
 import io.github.resilience4j.retry.Retry;
 import io.github.resilience4j.retry.RetryConfig;
@@ -13,6 +17,7 @@
 import org.slf4j.LoggerFactory;
 import org.slf4j.event.Level;
 
+import java.util.Locale;
 import java.util.function.Supplier;
 
 /**
@@ -52,6 +57,7 @@ public final class RetryHandler {
     private final int maxAttempts;
     private final int numFailsBeforeWarn;
     private final Retry retry;
+    private final Counter grpcRetryCounter;
 
     /**
      * Constructs the retry policy for a given gRPC service.
@@ -66,6 +72,14 @@ public RetryHandler(Configurator configG, String retryName) {
         maxAttempts = configG.findIntEntry(GRPC_RETRY_MAX_ATTEMPTS, 4);
         numFailsBeforeWarn = configG.findIntEntry(GRPC_RETRY_NUM_FAILS_BEFORE_WARN, 3);
 
+        // Initialize Dropwizard counter for gRPC retries
+        try {
+            grpcRetryCounter = MetricsManager.lookup().getMetricRegistry()
+                    .counter("grpc.retries." + sanitizeMetricName(retryName));
+        } catch (NamespaceException e) {
+            throw new VerifyException("Failed to initialize gRPC retry counter", e);
+        }
+
         retry = Retry.of(internalName, RetryConfig.custom()
                 .maxAttempts(maxAttempts)
                 .intervalFunction(IntervalFunction.ofExponentialBackoff(
@@ -79,7 +93,17 @@ public RetryHandler(Configurator configG, String retryName) {
                 .onError(this::logMessageOnError);
     }
 
+    /**
+     * Sanitizes metric names to be compatible with Prometheus naming conventions
+     */
+    private static String sanitizeMetricName(String name) {
+        return name.replaceAll("[^a-zA-Z0-9_]", "_").toLowerCase(Locale.ROOT);
+    }
+
     private void logMessageOnRetry(RetryOnRetryEvent event) {
+        // Increment the Dropwizard counter for each retry attempt
+        grpcRetryCounter.inc();
+
         int attemptNumber = event.getNumberOfRetryAttempts();
         Level level = attemptNumber <= numFailsBeforeWarn ? Level.INFO : Level.WARN;
         logger.atLevel(level).log("{} failed gRPC connection attempt #{} with event error: {}", internalName, attemptNumber, event);

src/main/java/emissary/server/EmissaryServer.java

@@ -627,7 +627,18 @@ private static ConstraintSecurityHandler buildSecurityHandler() {
         health.setPathSpec("/api/health");
         health.setConstraint(noAuthConstraint);
 
-        handler.setConstraintMappings(new ConstraintMapping[] {mapping, health});
+        // TODO: figure out how to allow Digest Authenticator to work with Prometheus
+
+        ConstraintMapping metrics = new ConstraintMapping();
+        metrics.setPathSpec("/api/metrics");
+        metrics.setConstraint(noAuthConstraint);
+
+        ConstraintMapping metricsPrometheus = new ConstraintMapping();
+        metricsPrometheus.setPathSpec("/api/metrics?format=prometheus");
+        metricsPrometheus.setConstraint(noAuthConstraint);
+
+        handler.setConstraintMappings(
+                new ConstraintMapping[] {mapping, health, metrics, metricsPrometheus});
         handler.setAuthenticator(new DigestAuthenticator());
         return handler;
     }

src/main/java/emissary/server/api/HealthCheckAction.java

@@ -3,14 +3,18 @@
 import emissary.core.MetricsManager;
 import emissary.core.NamespaceException;
 
+import com.codahale.metrics.health.HealthCheck;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Map;
+
 @Path("")
 // context is /api, set in EmissaryServer
 public class HealthCheckAction {
@@ -20,13 +24,56 @@ public class HealthCheckAction {
 
     @GET
     @Path("/" + HEALTH)
-    @Produces(MediaType.APPLICATION_JSON)
-    public Response clusterAgents() {
+    @Produces({MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN})
+    public Response healthCheck(@QueryParam("format") String format) {
         try {
-            return Response.ok().entity(MetricsManager.lookup().getHealthCheckRegistry().runHealthChecks()).build();
+            Map<String, HealthCheck.Result> results = MetricsManager.lookup().getHealthCheckRegistry().runHealthChecks();
+
+            if ("prometheus".equals(format)) {
+                return generatePrometheusResponse(results);
+            } else {
+                // Default to JSON format
+                return generateJsonResponse(results);
+            }
         } catch (NamespaceException ex) {
             logger.warn("Could not lookup MetricsManager", ex);
-            return Response.serverError().entity("Could not lookup MetricsManager").build();
+
+            if ("prometheus".equals(format)) {
+                return generatePrometheusErrorResponse();
+            } else {
+                return generateJsonErrorResponse();
+            }
+        }
+    }
+
+    protected Response generateJsonResponse(Map<String, HealthCheck.Result> results) {
+        return Response.ok().entity(results).build();
+    }
+
+    protected Response generateJsonErrorResponse() {
+        return Response.serverError().entity("Could not lookup MetricsManager").build();
+    }
+
+    private static Response generatePrometheusResponse(Map<String, HealthCheck.Result> results) {
+        boolean allHealthy = results.values().stream().allMatch(HealthCheck.Result::isHealthy);
+
+        StringBuilder metricsOutput = new StringBuilder();
+        metricsOutput.append("# HELP emissary_health_status Health status of emissary service (1=healthy, 0=unhealthy)\n");
+        metricsOutput.append("# TYPE emissary_health_status gauge\n");
+        metricsOutput.append("emissary_health_status ").append(allHealthy ? "1" : "0").append("\n");
+
+        if (allHealthy) {
+            return Response.ok(metricsOutput.toString()).build();
+        } else {
+            return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(metricsOutput.toString()).build();
         }
     }
+
+    private static Response generatePrometheusErrorResponse() {
+        StringBuilder errorMetrics = new StringBuilder();
+        errorMetrics.append("# HELP emissary_health_status Health status of emissary service (1=healthy, 0=unhealthy)\n");
+        errorMetrics.append("# TYPE emissary_health_status gauge\n");
+        errorMetrics.append("emissary_health_status 0\n");
+        return Response.status(Response.Status.SERVICE_UNAVAILABLE).entity(errorMetrics.toString()).build();
+    }
 }