feat: support kafka metrics exporter

automq-shell/build.gradle

@@ -65,4 +65,4 @@ jar {
     manifest {
         attributes 'Main-Class': 'com.automq.shell.AutoMQCLI'
     }
-}
+}

build.gradle

@@ -49,6 +49,7 @@ plugins {
   //  We are going to drop JDK8 support. Hence, the spotless is upgrade to newest version and be applied only if the build env is compatible with JDK 11.
   //  spotless 6.15.0+ has issue in runtime with JDK8 even through we define it with `apply:false`. see https://github.com/diffplug/spotless/issues/2156 for more details
   id 'com.diffplug.spotless' version "6.14.0" apply false
+  id('com.google.protobuf') version '0.9.4'
 }
 
 ext {
@@ -165,8 +166,8 @@ allprojects {
           // ZooKeeper (potentially older and containing CVEs)
           libs.nettyHandler,
           libs.nettyTransportNativeEpoll,
-	  // be explicit about the reload4j version instead of relying on the transitive versions
-	  libs.reload4j
+      // be explicit about the reload4j version instead of relying on the transitive versions
+      libs.reload4j
         )
       }
     }
@@ -731,7 +732,7 @@ subprojects {
       jacoco {
         toolVersion = versions.jacoco
       }
-      
+
       jacocoTestReport {
         dependsOn tasks.test
         sourceSets sourceSets.main
@@ -755,8 +756,8 @@ subprojects {
     skipProjects = [ ":jmh-benchmarks", ":trogdor" ]
     skipConfigurations = [ "zinc" ]
   }
-  //  the task `removeUnusedImports` is implemented by google-java-format, 
-  //  and unfortunately the google-java-format version used by spotless 6.14.0 can't work with JDK 21. 
+  //  the task `removeUnusedImports` is implemented by google-java-format,
+  //  and unfortunately the google-java-format version used by spotless 6.14.0 can't work with JDK 21.
   //  Hence, we apply spotless tasks only if the env is either JDK11 or JDK17
   if ((JavaVersion.current().isJava11() || (JavaVersion.current() == JavaVersion.VERSION_17))) {
     apply plugin: 'com.diffplug.spotless'
@@ -978,6 +979,10 @@ project(':core') {
     implementation libs.opentelemetryExporterOTLP
     implementation libs.opentelemetryJmx
     implementation libs.awsSdkAuth
+    implementation 'com.google.protobuf:protobuf-java:3.25.5'
+    implementation 'com.squareup.wire:wire-schema:4.9.1'
+    implementation 'com.squareup.wire:wire-runtime:4.9.1'
+    implementation 'com.google.api.grpc:proto-google-common-protos:2.52.0'
 
     implementation(libs.oshi) {
       exclude group: 'org.slf4j', module: '*'
@@ -1223,7 +1228,7 @@ project(':core') {
     //By default gradle does not handle test dependencies between the sub-projects
     //This line is to include clients project test jar to dependant-testlibs
     from (project(':clients').testJar ) { "$buildDir/dependant-testlibs" }
-    // log4j-appender is not in core dependencies, 
+    // log4j-appender is not in core dependencies,
     // so we add it to dependant-testlibs to avoid ClassNotFoundException in running kafka_log4j_appender.py
     from (project(':log4j-appender').jar ) { "$buildDir/dependant-testlibs" }
     duplicatesStrategy 'exclude'
@@ -1235,22 +1240,64 @@ project(':core') {
     configProperties = checkstyleConfigProperties("import-control-core.xml")
   }
 
+  apply plugin: 'com.google.protobuf'
+
+  protobuf {
+    protoc {
+      // The artifact spec for the Protobuf Compiler
+      artifact = 'com.google.protobuf:protoc:3.25.5'
+    }
+
+    generateProtoTasks {
+      all().each { task ->
+        task.builtins {
+          java {}
+        }
+      }
+    }
+  }
+
+  tasks.named('compileJava') {
+    dependsOn 'generateProto'
+  }
+
+  tasks.named('spotlessJava') {
+    dependsOn tasks.named('generateProto')
+  }
+
+  spotless {
+    java {
+      targetExclude(
+        "build/generated/source/proto/main/java/**/*.java"
+      )
+    }
+  }
+
   sourceSets {
     // Set java/scala source folders in the `scala` block to enable joint compilation
     main {
       java {
         srcDirs = []
       }
       scala {
-        srcDirs = ["src/generated/java", "src/main/java", "src/main/scala"]
+        srcDirs = [
+          "build/generated/source/proto/main/java",
+          "src/generated/java",
+          "src/main/java",
+          "src/main/scala"
+        ]
       }
     }
     test {
       java {
         srcDirs = []
       }
       scala {
-        srcDirs = ["src/test/java", "src/test/scala"]
+        srcDirs = [
+          "build/generated/source/proto/test/java",
+          "src/test/java",
+          "src/test/scala"
+        ]
       }
     }
   }
@@ -1483,7 +1530,7 @@ project(':transaction-coordinator') {
     implementation project(':clients')
     generator project(':generator')
   }
-  
+
   sourceSets {
     main {
       java {
@@ -3685,4 +3732,4 @@ task aggregatedJavadoc(type: Javadoc, dependsOn: compileJava) {
   classpath = files(projectsWithJavadoc.collect { it.sourceSets.main.compileClasspath })
   includes = projectsWithJavadoc.collectMany { it.javadoc.getIncludes() }
   excludes = projectsWithJavadoc.collectMany { it.javadoc.getExcludes() }
-}
+}

checkstyle/checkstyle.xml

@@ -27,6 +27,9 @@
 <!--  <module name="Header">-->
 <!--    <property name="headerFile" value="${config_loc}/java.header" />-->
 <!--  </module>-->
+    <module name="BeforeExecutionExclusionFileFilter">
+        <property name="fileNamePattern" value=".*[/\\]generated[/\\]source[/\\]proto[/\\]main[/\\]java[/\\].*"/>
+    </module>
 
   <module name="TreeWalker">
 
@@ -152,7 +155,7 @@
     <module name="SuppressWarningsHolder"/>
 
     <module name="ModifierOrder"/>
-    
+
   </module>
 
   <module name="SuppressionFilter">

core/src/main/proto/common/v1/common.proto

@@ -0,0 +1,81 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package kafka.automq.telemetry.proto.common.v1;
+
+option csharp_namespace = "OpenTelemetry.Proto.Common.V1";
+option java_multiple_files = true;
+option java_package = "kafka.automq.telemetry.proto.common.v1";
+option java_outer_classname = "CommonProto";
+option go_package = "go.opentelemetry.io/proto/otlp/common/v1";
+
+// AnyValue is used to represent any type of attribute value. AnyValue may contain a
+// primitive value such as a string or integer or it may contain an arbitrary nested
+// object containing arrays, key-value lists and primitives.
+message AnyValue {
+  // The value is one of the listed fields. It is valid for all values to be unspecified
+  // in which case this AnyValue is considered to be "empty".
+  oneof value {
+    string string_value = 1;
+    bool bool_value = 2;
+    int64 int_value = 3;
+    double double_value = 4;
+    ArrayValue array_value = 5;
+    KeyValueList kvlist_value = 6;
+    bytes bytes_value = 7;
+  }
+}
+
+// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message
+// since oneof in AnyValue does not allow repeated fields.
+message ArrayValue {
+  // Array of values. The array may be empty (contain 0 elements).
+  repeated AnyValue values = 1;
+}
+
+// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message
+// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need
+// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to
+// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches
+// are semantically equivalent.
+message KeyValueList {
+  // A collection of key/value pairs of key-value pairs. The list may be empty (may
+  // contain 0 elements).
+  // The keys MUST be unique (it is not allowed to have more than one
+  // value with the same key).
+  repeated KeyValue values = 1;
+}
+
+// KeyValue is a key-value pair that is used to store Span attributes, Link
+// attributes, etc.
+message KeyValue {
+  string key = 1;
+  AnyValue value = 2;
+}
+
+// InstrumentationScope is a message representing the instrumentation scope information
+// such as the fully qualified name and version.
+message InstrumentationScope {
+  // An empty instrumentation scope name means the name is unknown.
+  string name = 1;
+  string version = 2;
+
+  // Additional attributes that describe the scope. [Optional].
+  // Attribute keys MUST be unique (it is not allowed to have more than one
+  // attribute with the same key).
+  repeated KeyValue attributes = 3;
+  uint32 dropped_attributes_count = 4;
+}