Add support for jmx metrics and prometheus exporter

charts/trino/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.23.1
+version: 0.24.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/trino/README.md

@@ -1,6 +1,6 @@
 # trino
 
-![Version: 0.23.1](https://img.shields.io/badge/Version-0.23.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 449](https://img.shields.io/badge/AppVersion-449-informational?style=flat-square)
+![Version: 0.24.0](https://img.shields.io/badge/Version-0.24.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 449](https://img.shields.io/badge/AppVersion-449-informational?style=flat-square)
 
 Fast distributed SQL query engine for big data analytics that helps you explore your data universe
 
@@ -605,6 +605,50 @@ Fast distributed SQL query engine for big data analytics that helps you explore
        }
      }
   ```
+* `jmx.enabled` - bool, default: `false`  
+
+  Set to true to enable the RMI server to expose Trino's [JMX metrics](https://trino.io/docs/current/admin/jmx.html).
+* `jmx.registryPort` - int, default: `9080`
+* `jmx.serverPort` - int, default: `9081`
+* `jmx.exporter.enabled` - bool, default: `false`  
+
+  Set to true to export JMX Metrics via HTTP for [Prometheus](https://github.com/prometheus/jmx_exporter) consumption
+* `jmx.exporter.image` - string, default: `"bitnami/jmx-exporter:latest"`
+* `jmx.exporter.pullPolicy` - string, default: `"Always"`
+* `jmx.exporter.port` - int, default: `5556`
+* `jmx.exporter.configProperties` - list, default: `[]`  
+
+  JMX Config Properties is mounted to /etc/jmx-exporter/jmx-exporter-config.yaml
+  Example:
+  ```yaml
+   configProperties: |-
+      startDelaySeconds: 0
+      ssl: false
+      lowercaseOutputName: false
+      lowercaseOutputLabelNames: false
+      includeObjectNames: ["java.lang:type=Threading"]
+      autoExcludeObjectNameAttributes: true
+      excludeObjectNameAttributes:
+        "java.lang:type=OperatingSystem":
+          - "ObjectName"
+        "java.lang:type=Runtime":
+          - "ClassPath"
+          - "SystemProperties"
+      rules:
+        - pattern: 'java\.lang<type=Threading><(.*)>ThreadCount: (.*)'
+          name: java_lang_Threading_ThreadCount
+          value: '$2'
+          help: 'ThreadCount (java.lang<type=Threading><>ThreadCount)'
+          type: UNTYPED
+* `serviceMonitor.enabled` - bool, default: `false`  
+
+  Set to true to create resources for the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator).
+* `serviceMonitor.selector` - object, default: `{"prometheus":"kube-prometheus"}`  
+
+  Labels for serviceMonitor, so that Prometheus can select it
+* `serviceMonitor.interval` - string, default: `"30s"`  
+
+  The serviceMonitor web endpoint interval
 * `commonLabels` - object, default: `{}`  
 
   Labels that get applied to every resource's metadata

charts/trino/templates/configmap-coordinator.yaml

@@ -43,6 +43,9 @@ data:
   {{- range $configValue := .Values.coordinator.additionalJVMConfig }}
     {{ $configValue }}
   {{- end }}
+  {{- if .Values.jmx.enabled }}
+    -Dcom.sun.management.jmxremote.rmi.port={{ .Values.jmx.serverPort }}
+  {{- end }}
 
   config.properties: |
     coordinator=true
@@ -69,6 +72,10 @@ data:
     http-server.https.port={{ .Values.server.config.https.port }}
     http-server.https.keystore.path={{ .Values.server.config.https.keystore.path }}
     {{- end }}
+    {{- if .Values.jmx.enabled }}
+    jmx.rmiregistry.port={{ .Values.jmx.registryPort }}
+    jmx.rmiserver.port={{ .Values.jmx.serverPort }}
+    {{- end }}
     {{- if .Values.server.coordinatorExtraConfig }}
     {{- .Values.server.coordinatorExtraConfig | nindent 4 }}
     {{- end }}

charts/trino/templates/configmap-jmx-exporter.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.jmx.exporter.enabled -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "trino.fullname" . }}-jmx-exporter-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "trino.labels" . | nindent 4 }}
+    app.kubernetes.io/component: jmx
+data:
+  jmx-exporter-config.yaml: |-
+    {{- .Values.jmx.exporter.configProperties | nindent 4 }}
+{{- end }}

charts/trino/templates/deployment-coordinator.yaml

@@ -73,6 +73,11 @@ spec:
                 path: group.db
               {{- end }}
         {{- end }}
+        {{- if .Values.jmx.exporter.enabled }}
+        - name: jmx-exporter-config-volume
+          configMap:
+            name: {{ template "trino.fullname" . }}-jmx-exporter-config
+        {{- end }}
         {{- range .Values.configMounts }}
         - name: {{ .name }}
           configMap:
@@ -155,6 +160,19 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+            {{- if .Values.jmx.enabled }}
+            - name: jmx-registry
+              containerPort: {{ .Values.jmx.registryPort }}
+              protocol: TCP
+            - name: jmx-server
+              containerPort: {{ .Values.jmx.serverPort }}
+              protocol: TCP
+            {{- end }}
+            {{- if .Values.serviceMonitor.enabled }}
+            - name: jmx-exporter
+              containerPort: {{ .Values.jmx.exporter.port }}
+              protocol: TCP
+            {{- end }}
           {{- range $key, $value := .Values.coordinator.additionalExposedPorts }}
             - name: {{ $value.name }}
               containerPort: {{ $value.port }}
@@ -181,6 +199,17 @@ spec:
             {{- toYaml .Values.coordinator.lifecycle | nindent 12 }}
           resources:
             {{- toYaml .Values.coordinator.resources | nindent 12 }}
+      {{- if .Values.jmx.exporter.enabled }}
+        - name: jmx-exporter
+          image: {{ .Values.jmx.exporter.image }}
+          imagePullPolicy: {{ .Values.jmx.exporter.pullPolicy }}
+          args:
+            - "{{ .Values.jmx.exporter.port }}"
+            - /etc/jmx-exporter/jmx-exporter-config.yaml
+          volumeMounts:
+            - mountPath: /etc/jmx-exporter/
+              name: jmx-exporter-config-volume
+      {{- end }}
       {{- if .Values.sidecarContainers.coordinator }}
         {{- toYaml .Values.sidecarContainers.coordinator | nindent 8 }}
       {{- end }}

charts/trino/templates/service.yaml

@@ -14,6 +14,12 @@ spec:
       targetPort: http
       protocol: TCP
       name: http
+    {{- if .Values.jmx.exporter.enabled }}
+    - port: {{ .Values.jmx.exporter.port }}
+      targetPort: jmx-exporter
+      protocol: TCP
+      name: jmx-exporter
+    {{- end }}
     {{- range $key, $value := .Values.coordinator.additionalExposedPorts }}
     - port: {{ $value.servicePort }}
       name: {{ $value.name }}

charts/trino/templates/servicemonitor.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.serviceMonitor.enabled -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "trino.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    release: prometheus-operator
+    {{- include "trino.labels" . | nindent 4 }}
+    {{- if .Values.serviceMonitor.selector }}
+    {{- toYaml .Values.serviceMonitor.selector | nindent 4 }}
+    {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "trino.selectorLabels" . | nindent 6 }}
+  endpoints:
+    - port: jmx-exporter
+      interval: {{ .Values.serviceMonitor.interval }}
+{{- end }}

charts/trino/templates/tests/test-connection.yaml

@@ -24,8 +24,8 @@ spec:
       - --password
     {{- end }}
       - --debug
-      - --network-logging=BODY
       - --execute=SELECT 1
+      - --no-progress
     {{- if eq .Values.server.config.authenticationType "PASSWORD" }}
       env:
         - name: TRINO_PASSWORD

charts/trino/templates/tests/test-jmx.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.jmx.exporter.enabled }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ include "trino.fullname" . }}-test-jmx
+  labels:
+    {{- include "trino.labels" . | nindent 4 }}
+    app.kubernetes.io/component: test
+    test: jmx
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  containers:
+    {{- if .Values.jmx.exporter.enabled }}
+    - name: trino-jmx
+      image: {{ include "trino.image" . }}
+      command: ["/bin/bash", "-c"]
+      args:
+      - curl -s {{ include "trino.fullname" . }}.{{ .Release.Namespace }}:5556 | grep -q trino
+    {{- end }}
+    {{- if .Values.serviceMonitor.enabled }}
+    - name: podmonitor
+      image: {{ include "trino.image" . }}
+      command: ["/bin/bash", "-c"]
+      args:
+      - sleep 60
+      - |
+        output=$(curl -s "prometheus-operator-kube-p-prometheus.default:9090/api/v1/targets?scrapePool=serviceMonitor/{{ .Release.Namespace }}/{{ include "trino.fullname" . }}/0&state=active" |
+        python -c 'import sys, json; print(json.load(sys.stdin)["data"]["activeTargets"][0]["discoveredLabels"]["__meta_kubernetes_service_name"])') && [ "$output" == "{{ include "trino.fullname" . }}" ]
+    {{- end }}
+  restartPolicy: Never
+{{- end }}

charts/trino/values.yaml

@@ -727,6 +727,51 @@ kafka:
   #    }
   # ```
 
+jmx:
+  # -- Set to true to enable the RMI server to expose Trino's [JMX metrics](https://trino.io/docs/current/admin/jmx.html).
+  enabled: false
+  registryPort: 9080
+  serverPort: 9081
+  exporter:
+    # jmx.exporter.enabled -- Set to true to export JMX Metrics via HTTP for [Prometheus](https://github.com/prometheus/jmx_exporter) consumption
+    enabled: false
+    image: bitnami/jmx-exporter:latest
+    pullPolicy: Always
+    port: 5556
+    configProperties: []
+    # jmx.exporter.configProperties -- JMX Config Properties is mounted to /etc/jmx-exporter/jmx-exporter-config.yaml
+    # @raw
+    # Example:
+    # ```yaml
+    #  configProperties: |-
+    #     startDelaySeconds: 0
+    #     ssl: false
+    #     lowercaseOutputName: false
+    #     lowercaseOutputLabelNames: false
+    #     includeObjectNames: ["java.lang:type=Threading"]
+    #     autoExcludeObjectNameAttributes: true
+    #     excludeObjectNameAttributes:
+    #       "java.lang:type=OperatingSystem":
+    #         - "ObjectName"
+    #       "java.lang:type=Runtime":
+    #         - "ClassPath"
+    #         - "SystemProperties"
+    #     rules:
+    #       - pattern: 'java\.lang<type=Threading><(.*)>ThreadCount: (.*)'
+    #         name: java_lang_Threading_ThreadCount
+    #         value: '$2'
+    #         help: 'ThreadCount (java.lang<type=Threading><>ThreadCount)'
+    #         type: UNTYPED
+
+serviceMonitor:
+  # serviceMonitor.enabled -- Set to true to create resources for the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator).
+  enabled: false
+  # serviceMonitor.selector -- Labels for serviceMonitor, so that Prometheus can select it
+  selector:
+    prometheus: kube-prometheus
+  # serviceMonitor.interval -- The serviceMonitor web endpoint interval
+  interval: "30s"
+
 # -- Labels that get applied to every resource's metadata
 commonLabels: {}
 

test-values.yaml

@@ -140,6 +140,28 @@ accessControl:
         ]
       }
 
+jmx:
+  enabled: true
+  registryPort: 9080
+  serverPort: 9081
+  exporter:
+    enabled: true
+    image: bitnami/jmx-exporter:latest
+    pullPolicy: Always
+    port: 5556
+    configProperties: |
+      startDelaySeconds: 0
+      hostPort: 127.0.0.1:9080
+      rules:
+        - pattern: 'trino.memory*'
+        - pattern: 'trino.execution<name=QueryManager>*'
+
+serviceMonitor:
+  enabled: true
+  selector:
+    prometheus: default
+  interval: "30s"
+
 ingress:
   enabled: true
   hosts:

test.sh

@@ -16,6 +16,7 @@ function join_by {
     fi
 }
 
+
 # default to randomly generated namespace, same as chart-testing would do, but we need to load secrets into the same namespace
 NAMESPACE=trino-$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 6 || true)
 HELM_EXTRA_SET_ARGS=
@@ -74,6 +75,14 @@ openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
 kubectl create namespace "$NAMESPACE"
 kubectl -n "$NAMESPACE" create secret tls certificates --cert=cert.crt --key=cert.key
 
+# only install prometheus helm chart for these tests
+if printf '%s\0' "${TEST_NAMES[@]}" | grep -qwz complete_values; then
+    helm repo add "prometheus-community" "https://prometheus-community.github.io/helm-charts"
+    helm upgrade --install "prometheus-operator" "prometheus-community/kube-prometheus-stack" -n default --version "60.0.2"
+    kubectl rollout status --watch deployments -l release=prometheus-operator -n default
+fi
+
+
 CT_ARGS+=(--namespace "$NAMESPACE")
 
 result=0
@@ -84,7 +93,7 @@ for test_name in "${TEST_NAMES[@]}"; do
     if ! time ct install "${CT_ARGS[@]}" --helm-extra-set-args "$HELM_EXTRA_SET_ARGS ${testCases[$test_name]}"; then
         echo 1>&2 "❌ Test $test_name failed"
         echo 1>&2 "Test logs:"
-        kubectl --namespace "$NAMESPACE" logs --tail=-1 --selector app.kubernetes.io/component=test
+        kubectl --namespace "$NAMESPACE" logs --tail=-1 --selector app.kubernetes.io/component=test --all-containers=true
         result=1
     else
         echo 1>&2 "✅ Test $test_name completed"
@@ -99,6 +108,8 @@ done
 
 if [ "$CLEANUP_NAMESPACE" == "true" ]; then
     kubectl delete namespace "$NAMESPACE"
+    helm uninstall "prometheus-operator" --ignore-not-found
+    # TODO: delete CRDs https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#uninstall-helm-chart
 fi
 
 exit $result