Skip to content

Commit 562674d

Browse files
committed
Initial commmit
0 parents  commit 562674d

28 files changed

+1143
-0
lines changed

.gitignore

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
**/HELP.md
2+
**/.gradle
3+
**/build/
4+
**/!gradle/wrapper/gradle-wrapper.jar
5+
6+
### IntelliJ IDEA ###
7+
**/.idea
8+
**/*.iws
9+
**/*.iml
10+
**/*.ipr
11+
**/out/

docker-compose.yml

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
version: '2.1'
2+
3+
services:
4+
zoo1:
5+
image: zookeeper:3.4.9
6+
hostname: zoo1
7+
ports:
8+
- "2181:2181"
9+
environment:
10+
ZOO_MY_ID: 1
11+
ZOO_PORT: 2181
12+
ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
13+
volumes:
14+
- ./zk-multiple-kafka-multiple/zoo1/data:/data
15+
- ./zk-multiple-kafka-multiple/zoo1/datalog:/datalog
16+
17+
zoo2:
18+
image: zookeeper:3.4.9
19+
hostname: zoo2
20+
ports:
21+
- "2182:2182"
22+
environment:
23+
ZOO_MY_ID: 2
24+
ZOO_PORT: 2182
25+
ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
26+
volumes:
27+
- ./zk-multiple-kafka-multiple/zoo2/data:/data
28+
- ./zk-multiple-kafka-multiple/zoo2/datalog:/datalog
29+
30+
zoo3:
31+
image: zookeeper:3.4.9
32+
hostname: zoo3
33+
ports:
34+
- "2183:2183"
35+
environment:
36+
ZOO_MY_ID: 3
37+
ZOO_PORT: 2183
38+
ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
39+
volumes:
40+
- ./zk-multiple-kafka-multiple/zoo3/data:/data
41+
- ./zk-multiple-kafka-multiple/zoo3/datalog:/datalog
42+
43+
kafka1:
44+
image: confluentinc/cp-kafka:5.2.1
45+
hostname: kafka1
46+
ports:
47+
- "9092:9092"
48+
environment:
49+
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://192.168.1.25:9092
50+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
51+
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
52+
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
53+
KAFKA_BROKER_ID: 1
54+
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
55+
volumes:
56+
- ./zk-multiple-kafka-multiple/kafka1/data:/var/lib/kafka/data
57+
depends_on:
58+
- zoo1
59+
- zoo2
60+
- zoo3
61+
62+
kafka2:
63+
image: confluentinc/cp-kafka:5.2.1
64+
hostname: kafka2
65+
ports:
66+
- "9093:9093"
67+
environment:
68+
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://192.168.1.25:9093
69+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
70+
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
71+
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
72+
KAFKA_BROKER_ID: 2
73+
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
74+
volumes:
75+
- ./zk-multiple-kafka-multiple/kafka2/data:/var/lib/kafka/data
76+
depends_on:
77+
- zoo1
78+
- zoo2
79+
- zoo3
80+
81+
kafka3:
82+
image: confluentinc/cp-kafka:5.2.1
83+
hostname: kafka3
84+
ports:
85+
- "9094:9094"
86+
environment:
87+
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://192.168.1.25:9094
88+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
89+
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
90+
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
91+
KAFKA_BROKER_ID: 3
92+
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
93+
volumes:
94+
- ./zk-multiple-kafka-multiple/kafka3/data:/var/lib/kafka/data
95+
depends_on:
96+
- zoo1
97+
- zoo2
98+
- zoo3
99+
100+
elasticsearch:
101+
image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1
102+
environment:
103+
- discovery.type=single-node
104+
ports:
105+
- "9200:9200"
106+
- "9300:9300"
107+
kibana:
108+
image: docker.elastic.co/kibana/kibana:7.1.1
109+
ports:
110+
- "5601:5601"

kafka-consumer/build.gradle

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
plugins {
2+
id 'org.springframework.boot' version '2.1.5.RELEASE'
3+
id 'java'
4+
}
5+
6+
apply plugin: 'io.spring.dependency-management'
7+
8+
group = 'eu.macphail'
9+
version = '0.0.1-SNAPSHOT'
10+
sourceCompatibility = '1.8'
11+
ext['elasticsearch.version'] = '7.1.1'
12+
13+
configurations {
14+
developmentOnly
15+
runtimeClasspath {
16+
extendsFrom developmentOnly
17+
}
18+
compileOnly {
19+
extendsFrom annotationProcessor
20+
}
21+
}
22+
23+
repositories {
24+
mavenCentral()
25+
}
26+
27+
dependencies {
28+
implementation 'org.springframework.boot:spring-boot-starter'
29+
implementation 'org.springframework.boot:spring-boot-starter-web'
30+
implementation 'org.springframework.kafka:spring-kafka'
31+
implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.1.1'
32+
compileOnly 'org.projectlombok:lombok'
33+
developmentOnly 'org.springframework.boot:spring-boot-devtools'
34+
annotationProcessor 'org.projectlombok:lombok'
35+
testImplementation 'org.springframework.boot:spring-boot-starter-test'
36+
testImplementation 'org.springframework.kafka:spring-kafka-test'
37+
}
53.9 KB
Binary file not shown.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#Sat Jun 01 15:47:48 CEST 2019
2+
distributionBase=GRADLE_USER_HOME
3+
distributionPath=wrapper/dists
4+
zipStoreBase=GRADLE_USER_HOME
5+
zipStorePath=wrapper/dists
6+
distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip

kafka-consumer/gradlew

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#!/usr/bin/env sh
2+
3+
##############################################################################
4+
##
5+
## Gradle start up script for UN*X
6+
##
7+
##############################################################################
8+
9+
# Attempt to set APP_HOME
10+
# Resolve links: $0 may be a link
11+
PRG="$0"
12+
# Need this for relative symlinks.
13+
while [ -h "$PRG" ] ; do
14+
ls=`ls -ld "$PRG"`
15+
link=`expr "$ls" : '.*-> \(.*\)$'`
16+
if expr "$link" : '/.*' > /dev/null; then
17+
PRG="$link"
18+
else
19+
PRG=`dirname "$PRG"`"/$link"
20+
fi
21+
done
22+
SAVED="`pwd`"
23+
cd "`dirname \"$PRG\"`/" >/dev/null
24+
APP_HOME="`pwd -P`"
25+
cd "$SAVED" >/dev/null
26+
27+
APP_NAME="Gradle"
28+
APP_BASE_NAME=`basename "$0"`
29+
30+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31+
DEFAULT_JVM_OPTS='"-Xmx64m"'
32+
33+
# Use the maximum available, or set MAX_FD != -1 to use that value.
34+
MAX_FD="maximum"
35+
36+
warn () {
37+
echo "$*"
38+
}
39+
40+
die () {
41+
echo
42+
echo "$*"
43+
echo
44+
exit 1
45+
}
46+
47+
# OS specific support (must be 'true' or 'false').
48+
cygwin=false
49+
msys=false
50+
darwin=false
51+
nonstop=false
52+
case "`uname`" in
53+
CYGWIN* )
54+
cygwin=true
55+
;;
56+
Darwin* )
57+
darwin=true
58+
;;
59+
MINGW* )
60+
msys=true
61+
;;
62+
NONSTOP* )
63+
nonstop=true
64+
;;
65+
esac
66+
67+
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
68+
69+
# Determine the Java command to use to start the JVM.
70+
if [ -n "$JAVA_HOME" ] ; then
71+
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
72+
# IBM's JDK on AIX uses strange locations for the executables
73+
JAVACMD="$JAVA_HOME/jre/sh/java"
74+
else
75+
JAVACMD="$JAVA_HOME/bin/java"
76+
fi
77+
if [ ! -x "$JAVACMD" ] ; then
78+
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
79+
80+
Please set the JAVA_HOME variable in your environment to match the
81+
location of your Java installation."
82+
fi
83+
else
84+
JAVACMD="java"
85+
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
86+
87+
Please set the JAVA_HOME variable in your environment to match the
88+
location of your Java installation."
89+
fi
90+
91+
# Increase the maximum file descriptors if we can.
92+
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
93+
MAX_FD_LIMIT=`ulimit -H -n`
94+
if [ $? -eq 0 ] ; then
95+
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
96+
MAX_FD="$MAX_FD_LIMIT"
97+
fi
98+
ulimit -n $MAX_FD
99+
if [ $? -ne 0 ] ; then
100+
warn "Could not set maximum file descriptor limit: $MAX_FD"
101+
fi
102+
else
103+
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
104+
fi
105+
fi
106+
107+
# For Darwin, add options to specify how the application appears in the dock
108+
if $darwin; then
109+
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110+
fi
111+
112+
# For Cygwin, switch paths to Windows format before running java
113+
if $cygwin ; then
114+
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115+
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116+
JAVACMD=`cygpath --unix "$JAVACMD"`
117+
118+
# We build the pattern for arguments to be converted via cygpath
119+
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
120+
SEP=""
121+
for dir in $ROOTDIRSRAW ; do
122+
ROOTDIRS="$ROOTDIRS$SEP$dir"
123+
SEP="|"
124+
done
125+
OURCYGPATTERN="(^($ROOTDIRS))"
126+
# Add a user-defined pattern to the cygpath arguments
127+
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
128+
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
129+
fi
130+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
131+
i=0
132+
for arg in "$@" ; do
133+
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
134+
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
135+
136+
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
137+
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
138+
else
139+
eval `echo args$i`="\"$arg\""
140+
fi
141+
i=$((i+1))
142+
done
143+
case $i in
144+
(0) set -- ;;
145+
(1) set -- "$args0" ;;
146+
(2) set -- "$args0" "$args1" ;;
147+
(3) set -- "$args0" "$args1" "$args2" ;;
148+
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149+
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150+
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151+
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152+
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153+
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154+
esac
155+
fi
156+
157+
# Escape application args
158+
save () {
159+
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160+
echo " "
161+
}
162+
APP_ARGS=$(save "$@")
163+
164+
# Collect all arguments for the java command, following the shell quoting and substitution rules
165+
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166+
167+
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168+
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169+
cd "$(dirname "$0")"
170+
fi
171+
172+
exec "$JAVACMD" "$@"

0 commit comments

Comments
 (0)