Add Kafka module with KRaft configuration

guilleiguaran · guilleiguaran · commit 602f58b8014d · 2025-10-20T18:54:10.000-05:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ TestContainers is composed of modules for common databases and services. This ch
 
 - [ElasticsearchContainer](https://github.com/testcontainers/testcontainers-ruby/tree/main/elasticsearch/CHANGELOG.md)
 - [OpensearchContainer](https://github.com/testcontainers/testcontainers-ruby/tree/main/opensearch/CHANGELOG.md)
+- [KafkaContainer](https://github.com/testcontainers/testcontainers-ruby/tree/main/kafka/CHANGELOG.md)
 
 - [MariadbContainer](https://github.com/testcontainers/testcontainers-ruby/tree/main/mariadb/CHANGELOG.md)
 
diff --git a/Gemfile b/Gemfile
@@ -11,6 +11,7 @@ gem "testcontainers-redis", path: "./redis"
 gem "testcontainers-postgres", path: "./postgres"
 gem "testcontainers-nginx", path: "./nginx"
 gem "testcontainers-elasticsearch", path: "./elasticsearch"
+gem "testcontainers-kafka", path: "./kafka"
 gem "testcontainers-opensearch", path: "./opensearch"
 gem "testcontainers-mariadb", path: "./mariadb"
 gem "testcontainers-mongo", path: "./mongo"
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -24,6 +24,12 @@ PATH
     testcontainers-elasticsearch (0.2.0)
       testcontainers-core (~> 0.1)
 
+PATH
+  remote: kafka
+  specs:
+    testcontainers-kafka (0.1.0)
+      testcontainers-core (~> 0.1)
+
 PATH
   remote: mariadb
   specs:
@@ -216,6 +222,7 @@ DEPENDENCIES
   testcontainers-compose!
   testcontainers-core!
   testcontainers-elasticsearch!
+  testcontainers-kafka!
   testcontainers-mariadb!
   testcontainers-mongo!
   testcontainers-mysql!
diff --git a/Rakefile b/Rakefile
@@ -6,7 +6,7 @@ require "rake/testtask"
 require "yard"
 require "standard/rake"
 
-MODULES = %w[core mysql redis postgres nginx elasticsearch opensearch mariadb mongo redpanda]
+MODULES = %w[core mysql redis postgres nginx elasticsearch opensearch kafka mariadb mongo redpanda]
 
 %w[test standard].each do |task_name|
   desc "Run #{task_name} task for all projects"
diff --git a/kafka/CHANGELOG.md b/kafka/CHANGELOG.md
@@ -0,0 +1,6 @@
+## [Unreleased]
+
+### Added
+
+- Initial release of `testcontainers-kafka` providing a single-broker KRaft
+  Kafka container and convenience helpers.
diff --git a/kafka/Gemfile b/kafka/Gemfile
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in testcontainers-kafka.gemspec
+gemspec
+
+# Use the latest version of testcontainers-core from the local path
+gem "testcontainers-core", path: "../core"
diff --git a/kafka/Gemfile.lock b/kafka/Gemfile.lock
@@ -0,0 +1,99 @@
+PATH
+  remote: ../core
+  specs:
+    testcontainers-core (0.2.0)
+      base64 (~> 0.3)
+      docker-api (~> 2.4)
+      java-properties (~> 0.3.0)
+
+PATH
+  remote: .
+  specs:
+    testcontainers-kafka (0.1.0)
+      testcontainers-core (~> 0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    base64 (0.3.0)
+    docker-api (2.4.0)
+      excon (>= 0.64.0)
+      multi_json
+    excon (1.3.0)
+      logger
+    ffi (1.17.2-arm64-darwin)
+    java-properties (0.3.0)
+    json (2.15.1)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
+    mini_portile2 (2.8.9)
+    minitest (5.26.0)
+    minitest-hooks (1.5.2)
+      minitest (> 5.3)
+    multi_json (1.17.0)
+    parallel (1.27.0)
+    parser (3.3.9.0)
+      ast (~> 2.4.1)
+      racc
+    prism (1.6.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (13.3.0)
+    rdkafka (0.24.1-arm64-darwin)
+      ffi (~> 1.15)
+      json (> 2.0)
+      logger
+      mini_portile2 (~> 2.6)
+      rake (> 12)
+    regexp_parser (2.11.3)
+    rubocop (1.80.2)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.46.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.47.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-performance (1.25.0)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.38.0, < 2.0)
+    ruby-progressbar (1.13.0)
+    standard (1.51.1)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.80.2)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.8)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.8.0)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.25.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.1.0)
+
+PLATFORMS
+  arm64-darwin-23
+
+DEPENDENCIES
+  minitest (~> 5.0)
+  minitest-hooks (~> 1.5)
+  rake (~> 13.0)
+  rdkafka (~> 0.12)
+  standard (~> 1.3)
+  testcontainers-core!
+  testcontainers-kafka!
+
+BUNDLED WITH
+   2.6.9
diff --git a/kafka/LICENSE.txt b/kafka/LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Guillermo Iguaran
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/kafka/README.md b/kafka/README.md
@@ -0,0 +1,63 @@
+# Testcontainers module for Kafka
+
+## Installation
+
+Add the library to the test section in your application's Gemfile:
+
+```ruby
+group :test do
+  gem 'testcontainers-kafka'
+end
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install testcontainers-kafka
+```
+
+## Usage
+
+```ruby
+require "testcontainers/kafka"
+require "rdkafka"
+
+container = Testcontainers::KafkaContainer.new.start
+
+begin
+  container.wait_for_tcp_port(9092)
+  config = {
+    "bootstrap.servers": container.connection_url,
+    "group.id": "ruby-example",
+    "auto.offset.reset": "earliest"
+  }
+
+  producer = Rdkafka::Config.new(config).producer
+  producer.produce(payload: "hello", topic: "example").wait
+
+  consumer = Rdkafka::Config.new(config).consumer
+  consumer.subscribe("example")
+  puts consumer.each { |msg| break msg.payload }
+ensure
+  container.stop if container.running?
+  container.remove if container.exists?
+end
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/testcontainers/testcontainers-ruby. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/testcontainers/testcontainers-ruby/blob/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Testcontainers project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/testcontainers/testcontainers-ruby/blob/main/CODE_OF_CONDUCT.md).
diff --git a/kafka/Rakefile b/kafka/Rakefile
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+require "standard/rake"
+
+task default: %i[test standard]
diff --git a/kafka/lib/testcontainers/kafka.rb b/kafka/lib/testcontainers/kafka.rb
@@ -0,0 +1,102 @@
+require_relative "kafka/version"
+require "testcontainers"
+require "stringio"
+
+module Testcontainers
+  # KafkaContainer runs a single-node Kafka broker using KRaft (no ZooKeeper).
+  class KafkaContainer < ::Testcontainers::DockerContainer
+    KAFKA_DEFAULT_IMAGE = "confluentinc/cp-kafka:7.6.1"
+    KAFKA_DEFAULT_PORT = 9092
+    KAFKA_DEFAULT_CONTROLLER_PORT = 9093
+
+    attr_reader :kafka_port, :controller_port, :host_kafka_port
+
+    def initialize(image = KAFKA_DEFAULT_IMAGE, kafka_port: KAFKA_DEFAULT_PORT, controller_port: KAFKA_DEFAULT_CONTROLLER_PORT, host_kafka_port: KAFKA_DEFAULT_PORT, **kwargs)
+      @kafka_port = kafka_port
+      @controller_port = controller_port
+      @host_kafka_port = host_kafka_port
+      super(image, **kwargs)
+      configure_environment
+      add_wait_for(:tcp_port, kafka_port, timeout: 180, interval: 1) unless wait_for_user_defined?
+    end
+
+    def start
+      with_fixed_exposed_port(@kafka_port, @host_kafka_port)
+      with_exposed_ports(@controller_port)
+      with_entrypoint(%w[/bin/bash])
+      with_command(["-c", "while [ ! -f #{startup_script_path} ]; do sleep 0.1; done; #{startup_script_path}"])
+      super
+
+      copy_file_to_container("/tmp" + startup_script_path, startup_script)
+      exec_as_root(%w[chmod 755] + ["/tmp" + startup_script_path])
+      exec_as_root(%w[mv] + ["/tmp" + startup_script_path, startup_script_path])
+      self
+    end
+
+    def port
+      kafka_port
+    end
+
+    def connection_url
+      "#{host}:#{mapped_port(kafka_port)}"
+    end
+
+    def bootstrap_servers
+      "PLAINTEXT://#{connection_url}"
+    end
+
+    private
+
+    def configure_environment
+      add_env("KAFKA_BROKER_ID", "1")
+      add_env("KAFKA_NODE_ID", "1")
+      add_env("KAFKA_PROCESS_ROLES", "broker,controller")
+      add_env("KAFKA_CONTROLLER_LISTENER_NAMES", "CONTROLLER")
+      add_env("KAFKA_LISTENERS", "PLAINTEXT://0.0.0.0:#{kafka_port},CONTROLLER://0.0.0.0:#{controller_port}")
+      add_env("KAFKA_ADVERTISED_LISTENERS", "PLAINTEXT://localhost:#{host_kafka_port}")
+      add_env("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT")
+      add_env("KAFKA_CONTROLLER_QUORUM_VOTERS", "1@127.0.0.1:#{controller_port}")
+      add_env("KAFKA_INTER_BROKER_LISTENER_NAME", "PLAINTEXT")
+      add_env("KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS", "0")
+      add_env("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
+      add_env("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1")
+      add_env("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR", "1")
+      add_env("KAFKA_NUM_PARTITIONS", "1")
+      add_env("KAFKA_JVM_PERFORMANCE_OPTS", "-server -XX:+UseG1GC -Xms512m -Xmx512m")
+      add_env("CLUSTER_ID", "B3lQc2gyQlNKRzh5TUpTRw==")
+      add_env("KAFKA_CFG_PROCESS_ROLES", "broker,controller")
+      add_env("KAFKA_CFG_CONTROLLER_LISTENER_NAMES", "CONTROLLER")
+      add_env("KAFKA_CFG_LISTENERS", "PLAINTEXT://0.0.0.0:#{kafka_port},CONTROLLER://0.0.0.0:#{controller_port}")
+      add_env("KAFKA_CFG_ADVERTISED_LISTENERS", "PLAINTEXT://localhost:#{host_kafka_port}")
+      add_env("KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP", "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT")
+      add_env("KAFKA_CFG_CONTROLLER_QUORUM_VOTERS", "1@127.0.0.1:#{controller_port}")
+      add_env("KAFKA_CFG_INTER_BROKER_LISTENER_NAME", "PLAINTEXT")
+      add_env("KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS", "0")
+      add_env("KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR", "1")
+      add_env("KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR", "1")
+      add_env("KAFKA_CFG_NUM_PARTITIONS", "1")
+    end
+
+    def startup_script_path
+      "/testcontainers-kafka-start.sh"
+    end
+
+    def startup_script
+      script = StringIO.new
+      script.puts "#!/bin/bash"
+      script.puts "set -euo pipefail"
+      script.puts "CLUSTER_ID=${CLUSTER_ID:-'4L6g3n4kTAW5y-5Pb0jzGw'}"
+      script.puts "KAFKA_LOG_DIR=/var/lib/kafka/data"
+      script.puts "CONFIG_FILE=/etc/kafka/kraft/server.properties"
+      script.puts "if [ ! -f \"$KAFKA_LOG_DIR/meta.properties\" ]; then"
+      script.puts "  kafka-storage format --config $CONFIG_FILE --cluster-id $CLUSTER_ID --ignore-formatted"
+      script.puts "fi"
+      script.puts "exec /etc/confluent/docker/run"
+      script
+    end
+
+    def exec_as_root(cmd, options = {})
+      exec(cmd, options.merge({"User" => "root"}))
+    end
+  end
+end
diff --git a/kafka/lib/testcontainers/kafka/version.rb b/kafka/lib/testcontainers/kafka/version.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Testcontainers
+  module Kafka
+    VERSION = "0.1.0"
+  end
+end
diff --git a/kafka/test/kafka_container_test.rb b/kafka/test/kafka_container_test.rb
diff --git a/kafka/test/test_helper.rb b/kafka/test/test_helper.rb
diff --git a/kafka/testcontainers-kafka.gemspec b/kafka/testcontainers-kafka.gemspec
