Skip to content

Commit ee6fc2d

Browse files
sanks011sanks011
committed
Ensure docker-compose-cluster.yaml supports proper 3-node cluster deployment
1 parent c608789 commit ee6fc2d

File tree

1 file changed

+235
-0
lines changed

1 file changed

+235
-0
lines changed

‎docker/docker-compose-cluster.yaml

+235
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Three-node AutoMQ cluster setup with MinIO for production-like environments
17+
version: "3.8"
18+
19+
services:
20+
# MinIO service for S3 storage
21+
minio:
22+
container_name: "${MINIO_DOCKER_NAME-minio}"
23+
hostname: "${MINIO_DOCKER_NAME-minio}"
24+
image: minio/minio:RELEASE.2023-09-04T19-57-37Z
25+
ports:
26+
- "9000:9000" # MinIO API
27+
- "9001:9001" # MinIO Console
28+
environment:
29+
- MINIO_ROOT_USER=minioadmin
30+
- MINIO_ROOT_PASSWORD=minioadmin
31+
command: server /data --console-address ":9001"
32+
volumes:
33+
- minio_data:/data
34+
networks:
35+
automq_net:
36+
ipv4_address: 10.6.0.2
37+
healthcheck:
38+
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
39+
interval: 5s
40+
timeout: 5s
41+
retries: 3
42+
43+
# Create needed buckets
44+
mc:
45+
container_name: "${MC_DOCKER_NAME-mc}"
46+
hostname: "${MC_DOCKER_NAME-mc}"
47+
image: minio/mc:RELEASE.2023-09-07T22-48-55Z
48+
depends_on:
49+
minio:
50+
condition: service_healthy
51+
entrypoint: >
52+
/bin/sh -c "
53+
/usr/bin/mc config host add myminio http://minio:9000 minioadmin minioadmin;
54+
/usr/bin/mc mb myminio/automq-data;
55+
/usr/bin/mc policy set public myminio/automq-data;
56+
exit 0;
57+
"
58+
networks:
59+
- automq_net
60+
61+
# Controller node 1 (first controller in the cluster)
62+
controller1:
63+
container_name: "${CONTROLLER1_DOCKER_NAME-controller1}"
64+
hostname: "${CONTROLLER1_DOCKER_NAME-controller1}"
65+
stop_grace_period: 2m
66+
image: automqinc/automq:latest
67+
ports:
68+
- "9093:9093" # Controller API
69+
environment:
70+
- KAFKA_S3_ACCESS_KEY=minioadmin
71+
- KAFKA_S3_SECRET_KEY=minioadmin
72+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
73+
- KAFKA_CFG_LISTENERS=CONTROLLER://:9093
74+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT
75+
command:
76+
- bash
77+
- -c
78+
- |
79+
/opt/kafka/scripts/start.sh up --process.roles controller --node.id 0 --controller.quorum.voters 0@controller1:9093,1@controller2:9093,2@controller3:9093 --s3.data.buckets 0@s3://automq-data?region=us-east-1 --s3.ops.buckets 1@s3://automq-data?region=us-east-1 --s3.wal.path 0@s3://automq-data?region=us-east-1 --s3.endpoint http://10.6.0.2:9000 --s3.path-style-access true
80+
networks:
81+
automq_net:
82+
ipv4_address: 10.6.0.3
83+
depends_on:
84+
- minio
85+
- mc
86+
87+
# Controller node 2
88+
controller2:
89+
container_name: "${CONTROLLER2_DOCKER_NAME-controller2}"
90+
hostname: "${CONTROLLER2_DOCKER_NAME-controller2}"
91+
stop_grace_period: 2m
92+
image: automqinc/automq:latest
93+
ports:
94+
- "9094:9093" # Controller API (mapped to different port on host)
95+
environment:
96+
- KAFKA_S3_ACCESS_KEY=minioadmin
97+
- KAFKA_S3_SECRET_KEY=minioadmin
98+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
99+
- KAFKA_CFG_LISTENERS=CONTROLLER://:9093
100+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT
101+
command:
102+
- bash
103+
- -c
104+
- |
105+
/opt/kafka/scripts/start.sh up --process.roles controller --node.id 1 --controller.quorum.voters 0@controller1:9093,1@controller2:9093,2@controller3:9093 --s3.data.buckets 0@s3://automq-data?region=us-east-1 --s3.ops.buckets 1@s3://automq-data?region=us-east-1 --s3.wal.path 0@s3://automq-data?region=us-east-1 --s3.endpoint http://10.6.0.2:9000 --s3.path-style-access true
106+
networks:
107+
automq_net:
108+
ipv4_address: 10.6.0.4
109+
depends_on:
110+
- controller1
111+
112+
# Controller node 3
113+
controller3:
114+
container_name: "${CONTROLLER3_DOCKER_NAME-controller3}"
115+
hostname: "${CONTROLLER3_DOCKER_NAME-controller3}"
116+
stop_grace_period: 2m
117+
image: automqinc/automq:latest
118+
ports:
119+
- "9095:9093" # Controller API (mapped to different port on host)
120+
environment:
121+
- KAFKA_S3_ACCESS_KEY=minioadmin
122+
- KAFKA_S3_SECRET_KEY=minioadmin
123+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
124+
- KAFKA_CFG_LISTENERS=CONTROLLER://:9093
125+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT
126+
command:
127+
- bash
128+
- -c
129+
- |
130+
/opt/kafka/scripts/start.sh up --process.roles controller --node.id 2 --controller.quorum.voters 0@controller1:9093,1@controller2:9093,2@controller3:9093 --s3.data.buckets 0@s3://automq-data?region=us-east-1 --s3.ops.buckets 1@s3://automq-data?region=us-east-1 --s3.wal.path 0@s3://automq-data?region=us-east-1 --s3.endpoint http://10.6.0.2:9000 --s3.path-style-access true
131+
networks:
132+
automq_net:
133+
ipv4_address: 10.6.0.5
134+
depends_on:
135+
- controller2
136+
137+
# Broker node 1
138+
broker1:
139+
container_name: "${BROKER1_DOCKER_NAME-broker1}"
140+
hostname: "${BROKER1_DOCKER_NAME-broker1}"
141+
stop_grace_period: 2m
142+
image: automqinc/automq:latest
143+
ports:
144+
- "9092:9092" # Kafka API
145+
environment:
146+
- KAFKA_S3_ACCESS_KEY=minioadmin
147+
- KAFKA_S3_SECRET_KEY=minioadmin
148+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
149+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_IN_CAPACITY=5120
150+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_OUT_CAPACITY=5120
151+
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
152+
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092
153+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
154+
command:
155+
- bash
156+
- -c
157+
- |
158+
/opt/kafka/scripts/start.sh up --process.roles broker --node.id 3 --controller.quorum.voters 0@controller1:9093,1@controller2:9093,2@controller3:9093 --s3.data.buckets 0@s3://automq-data?region=us-east-1 --s3.ops.buckets 1@s3://automq-data?region=us-east-1 --s3.wal.path 0@s3://automq-data?region=us-east-1 --s3.endpoint http://10.6.0.2:9000 --s3.path-style-access true
159+
networks:
160+
automq_net:
161+
ipv4_address: 10.6.0.6
162+
depends_on:
163+
- controller1
164+
- controller2
165+
- controller3
166+
167+
# Broker node 2
168+
broker2:
169+
container_name: "${BROKER2_DOCKER_NAME-broker2}"
170+
hostname: "${BROKER2_DOCKER_NAME-broker2}"
171+
stop_grace_period: 2m
172+
image: automqinc/automq:latest
173+
ports:
174+
- "9096:9092" # Kafka API (mapped to different port on host)
175+
environment:
176+
- KAFKA_S3_ACCESS_KEY=minioadmin
177+
- KAFKA_S3_SECRET_KEY=minioadmin
178+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
179+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_IN_CAPACITY=5120
180+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_OUT_CAPACITY=5120
181+
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
182+
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9096
183+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
184+
command:
185+
- bash
186+
- -c
187+
- |
188+
/opt/kafka/scripts/start.sh up --process.roles broker --node.id 4 --controller.quorum.voters 0@controller1:9093,1@controller2:9093,2@controller3:9093 --s3.data.buckets 0@s3://automq-data?region=us-east-1 --s3.ops.buckets 1@s3://automq-data?region=us-east-1 --s3.wal.path 0@s3://automq-data?region=us-east-1 --s3.endpoint http://10.6.0.2:9000 --s3.path-style-access true
189+
networks:
190+
automq_net:
191+
ipv4_address: 10.6.0.7
192+
depends_on:
193+
- broker1
194+
195+
# Broker node 3
196+
broker3:
197+
container_name: "${BROKER3_DOCKER_NAME-broker3}"
198+
hostname: "${BROKER3_DOCKER_NAME-broker3}"
199+
stop_grace_period: 2m
200+
image: automqinc/automq:latest
201+
ports:
202+
- "9097:9092" # Kafka API (mapped to different port on host)
203+
environment:
204+
- KAFKA_S3_ACCESS_KEY=minioadmin
205+
- KAFKA_S3_SECRET_KEY=minioadmin
206+
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m -XX:MaxDirectMemorySize=1G
207+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_IN_CAPACITY=5120
208+
- KAFKA_CFG_AUTOBALANCER_REPORTER_NETWORK_OUT_CAPACITY=5120
209+
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
210+
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9097
211+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT
212+
command:
213+
- bash
214+
- -c
215+
- |
216+
/opt/kafka/scripts/start.sh up --process.roles broker --node.id 5 --controller.quorum.voters 0@controller1:9093,1@controller2:9093,2@controller3:9093 --s3.data.buckets 0@s3://automq-data?region=us-east-1 --s3.ops.buckets 1@s3://automq-data?region=us-east-1 --s3.wal.path 0@s3://automq-data?region=us-east-1 --s3.endpoint http://10.6.0.2:9000 --s3.path-style-access true
217+
networks:
218+
automq_net:
219+
ipv4_address: 10.6.0.8
220+
depends_on:
221+
- broker2
222+
223+
volumes:
224+
minio_data:
225+
driver: local
226+
227+
networks:
228+
automq_net:
229+
name: automq_net
230+
driver: bridge
231+
ipam:
232+
driver: default
233+
config:
234+
- subnet: "10.6.0.0/16"
235+
gateway: "10.6.0.1"

0 commit comments

Comments
 (0)