OffsetCommitService does not work with SSL cluster #402
Description
Hello,
I can not make the service OffsetCommitService work with a cluster secured with SSL.
What is strange is that other services work with the same certificates to write and read message, and to create and delete topics too.
Has this problem already appeared before ?
I use XInfraMonitor 2.5.16, Kafka 3.4.0 and Zookeeper 3.8.1
Please find below my configuration of xinframonitor. As you can see, all services use same SSL properties and files to connect.
Configuration
{ "single-cluster-monitor-sd": { "class.name": "com.linkedin.xinfra.monitor.apps.SingleClusterMonitor", "topic": "xinfra-monitor-topic", "zookeeper.connect": "UselessProperty:2181", "bootstrap.servers": "kfkr:8081,kfkn:8081", "request.timeout.ms": 9000, "produce.record.delay.ms": 100, "topic-management.topicManagementEnabled": true, "topic-management.topicCreationEnabled": true, "topic-management.replicationFactor": 4, "topic-management.partitionsToBrokersRatio": 2.0, "topic-management.rebalance.interval.ms": 600000, "topic-management.preferred.leader.election.check.interval.ms": 300000, "security.protocol": "SSL", "ssl.keystore.type": "PKCS12", "ssl.truststore.location": "/tls/truststore.jks", "ssl.truststore.password": "gsergersgers564894hersdt", "ssl.truststore.type": "PKCS12", "ssl.keystore.location": "/tls/keystore.jks", "ssl.keystore.password": "gsergersgers564894hersdt", "topic-management.topicFactory.props": {}, "topic-management.topic.props": { "retention.ms": "3600000" }, "produce.producer.props": { "client.id": "kmf-client-id-sd", "security.protocol": "SSL", "ssl.truststore.type": "PKCS12", "ssl.truststore.location": "/tls/truststore.jks", "ssl.truststore.password": "gsergersgers564894hersdt", "ssl.keystore.location": "/tls/keystore.jks", "ssl.keystore.type": "PKCS12", "ssl.keystore.password": "gsergersgers564894hersdt" }, "consume.latency.sla.ms": "20000", "consume.consumer.props": { "client.id": "kmf-client-id-sd", "security.protocol": "SSL", "ssl.truststore.location": "/tls/truststore.jks", "ssl.truststore.type": "PKCS12", "ssl.truststore.password": "gsergersgers564894hersdt", "ssl.keystore.location": "/tls/keystore.jks", "ssl.keystore.type": "PKCS12", "ssl.keystore.password": "gsergersgers564894hersdt" } }, "offset-commit-service-sd": { "security.protocol": "SSL", "ssl.truststore.location": "/tls/truststore.jks", "ssl.truststore.password": "gsergersgers564894hersdt", "ssl.keystore.type": "PKCS12", "ssl.keystore.location": "/tls/keystore.jks", "ssl.keystore.password": "gsergersgers564894hersdt", "ssl.truststore.type": "PKCS12", "class.name": "com.linkedin.xinfra.monitor.services.OffsetCommitService", "zookeeper.connect": "UselessProperty:2181", "bootstrap.servers": "kfkr:8081,kfkn:8081", "consumer.props": { "group.id": "target-consumer-group-sd" } }, "reporter-service-sd": { "security.protocol": "SSL", "ssl.truststore.type": "PKCS12", "ssl.truststore.location": "/tls/truststore.jks", "ssl.truststore.password": "gsergersgers564894hersdt", "ssl.keystore.type": "PKCS12", "ssl.keystore.location": "/tls/keystore.jks", "ssl.keystore.password": "gsergersgers564894hersdt", "class.name": "com.linkedin.xinfra.monitor.services.DefaultMetricsReporterService", "report.interval.sec": 1, "report.metrics.list": [ "kmf:type=kafka-monitor:offline-runnable-count", "kmf.services:type=produce-service,name=*:produce-availability-avg", "kmf.services:type=consume-service,name=*:consume-availability-avg", "kmf.services:type=produce-service,name=*:records-produced-total", "kmf.services:type=consume-service,name=*:records-consumed-total", "kmf.services:type=produce-service,name=*:records-produced-rate", "kmf.services:type=produce-service,name=*:produce-error-rate", "kmf.services:type=consume-service,name=*:consume-error-rate", "kmf.services:type=consume-service,name=*:records-lost-total", "kmf.services:type=consume-service,name=*:records-lost-rate", "kmf.services:type=consume-service,name=*:records-duplicated-total", "kmf.services:type=consume-service,name=*:records-delay-ms-avg", "kmf.services:type=commit-availability-service,name=*:offsets-committed-avg", "kmf.services:type=commit-availability-service,name=*:offsets-committed-total", "kmf.services:type=commit-availability-service,name=*:failed-commit-offsets-avg", "kmf.services:type=commit-availability-service,name=*:failed-commit-offsets-total", "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-avg", "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-max", "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-99th", "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-999th", "kmf.services:type=commit-latency-service,name=*:commit-offset-latency-ms-9999th", "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-creation-metadata-propagation-ms-avg", "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-creation-metadata-propagation-ms-max", "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-deletion-metadata-propagation-ms-avg", "kmf.services:type=cluster-topic-manipulation-service,name=*:topic-deletion-metadata-propagation-ms-max", "kmf.services:type=offset-commit-service,name=*:offset-commit-availability-avg", "kmf.services:type=offset-commit-service,name=*:offset-commit-service-success-rate", "kmf.services:type=offset-commit-service,name=*:offset-commit-service-success-total", "kmf.services:type=offset-commit-service,name=*:offset-commit-service-failure-rate", "kmf.services:type=offset-commit-service,name=*:offset-commit-service-failure-total" ] }, "cluster-topic-manipulation-service-sd": { "security.protocol": "SSL", "ssl.truststore.type": "PKCS12", "ssl.truststore.location": "/tls/truststore.jks", "ssl.truststore.password": "gsergersgers564894hersdt", "ssl.keystore.type": "PKCS12", "ssl.keystore.location": "/tls/keystore.jks", "ssl.keystore.password": "gsergersgers564894hersdt", "class.name": "com.linkedin.xinfra.monitor.services.ClusterTopicManipulationService", "zookeeper.connect": "UselessProperty:2181", "bootstrap.servers": "kfkr:8081,kfkn:8081", "topic": "xinfra-monitor-topic" }
I looked in the code and added some logs. From what I understand the problem occurs in OffsetCommitService, line 223 :
boolean consumerNetworkClientPollResult =
consumerNetworkClient.poll(future, _time.timer(Duration.ofSeconds(5).toMillis()));
LOGGER.debug("result of poll {}", consumerNetworkClientPollResult);
where the thread seems to be blocked.
In Kafka logs, I see the error message :
[2023-10-16 16:46:54,386] INFO [SocketServer listenerType=ZK_BROKER, nodeId=101] Failed authentication with /X.X.X.X (channelId=X.X.X.X:8083-X.X.X.X:65438-32280) (SSL handshake failed) (org.apache.kafka.common.network.Selector)
I tried to update dependencies 'org.apache.kafka.kafka_2.12 and org.apache.kafka.kafka-clients, but with no result