Skip to content

Commit aac3892

Browse files
fernandoraccaFernando Racca Braskic
fernandoracca
and
Fernando Racca Braskic
authored
K8s Services and deployments (#15)
Co-authored-by: Fernando Racca Braskic <[email protected]>
1 parent 23982c2 commit aac3892

27 files changed

+257
-35
lines changed

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ The main benefits of this small cluster is that it's easy to configure to run in
1919

2020
| Virgo cluster | Hadoop | Spark | Hive | Postgres | Livy |
2121
| ------------- |:--------:| ------:|-------|----------|----------|
22+
| 0.8.2 | 2.7.7 | *2.3.0* | 1.2.2 | 11 | Moved |
2223
| 0.7.5 | 2.7.7 | *2.3.0* | 1.2.2 | 9.5 | Moved |
2324
| 0.7.0 | 2.7.7 | *2.3.0* | 1.2.2 | 9.5 | 0.4 |
2425
| 0.6.2 | 2.7.7 | 2.2.3 | 1.2.2 | 9.5 | *0.4* |

‎change-version.sh

+3
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ echo "Changing from $OLD_VERSION to $VERSION"
99
find . -name Dockerfile -type f -exec sed -i "s/${OLD_VERSION}/${VERSION}/g" {} \;
1010
sed -i "s/${OLD_VERSION}/${VERSION}/g" docker-compose.yml
1111

12+
# YAMLs
13+
find "k8s/" -name "*.yaml" -type f -exec sed -i "s/${OLD_VERSION}/${VERSION}/g" {} \;
14+
1215
echo "Changes:"
1316
find . -name Dockerfile | xargs grep $VERSION
1417
grep $VERSION docker-compose.yml

‎config.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env bash
22

33
#USER Config
4-
VERSION=0.8.1
4+
VERSION=0.8.2
55
NETWORK_NAME=virgo
66
NETWORK_SUBNET=172.15.0.0/16
77

‎docker-compose.yml

+20-20
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
version: '2.4'
1+
version: '2'
22

33
services:
44
hadoop-namenode:
55
container_name: hadoop-namenode
6-
image: 'aiurtech/hadoop-namenode:0.8.1'
6+
image: 'aiurtech/hadoop-namenode:0.8.2'
77
networks:
88
privnet:
99
ipv4_address: 172.15.0.2
@@ -13,12 +13,12 @@ services:
1313
- 8020:8020
1414
- 50070:50070
1515
hostname: hadoop-namenode
16-
cpus: '2'
16+
# cpus: '2'
1717
mem_limit: 500m
1818
mem_reservation: 300m
1919
hadoop-datanode:
2020
container_name: hadoop-datanode
21-
image: 'aiurtech/hadoop-datanode:0.8.1'
21+
image: 'aiurtech/hadoop-datanode:0.8.2'
2222
networks:
2323
privnet:
2424
ipv4_address: 172.15.0.20
@@ -28,14 +28,14 @@ services:
2828
- 50010:50010
2929
- 50075:50075
3030
hostname: hadoop-datanode
31-
cpus: '2'
31+
#cpus: '2'
3232
mem_limit: 600m
3333
mem_reservation: 300m
3434
depends_on:
3535
- hadoop-namenode
3636
yarn-resourcemanager:
3737
container_name: yarn-resourcemanager
38-
image: 'aiurtech/yarn-resourcemanager:0.8.1'
38+
image: 'aiurtech/yarn-resourcemanager:0.8.2'
3939
networks:
4040
privnet:
4141
ipv4_address: 172.15.0.3
@@ -46,12 +46,12 @@ services:
4646
- 8033:8033
4747
- 8088:8088
4848
hostname: yarn-resourcemanager
49-
cpus: '2'
49+
# cpus: '2'
5050
mem_limit: 500m
5151
mem_reservation: 300m
5252
yarn-nodemanager:
5353
container_name: yarn-nodemanager
54-
image: 'aiurtech/yarn-nodemanager:0.8.1'
54+
image: 'aiurtech/yarn-nodemanager:0.8.2'
5555
networks:
5656
privnet:
5757
ipv4_address: 172.15.0.4
@@ -64,40 +64,40 @@ services:
6464
- yarn-resourcemanager
6565
hive-metastore-postgresql:
6666
container_name: hive-metastore-postgresql
67-
image: 'aiurtech/hive-metastore-postgresql:0.8.1'
67+
image: 'aiurtech/hive-metastore-postgresql:0.8.2'
6868
networks:
6969
privnet:
7070
ipv4_address: 172.15.0.5
7171
ports:
7272
- 5432:5432
7373
hostname: hive-metastore-postgresql
74-
cpus: '2'
74+
# cpus: '2'
7575
mem_limit: 500m
7676
mem_reservation: 300m
7777
hive-metastore:
7878
container_name: hive-metastore
79-
image: 'aiurtech/hive-metastore:0.8.1'
79+
image: 'aiurtech/hive-metastore:0.8.2'
8080
networks:
8181
privnet:
8282
ipv4_address: 172.15.0.10
8383
ports:
8484
- 9083:9083
8585
hostname: hive-metastore
86-
cpus: '2'
86+
# cpus: '2'
8787
mem_limit: 600m
8888
mem_reservation: 300m
8989
depends_on:
9090
- hive-metastore-postgresql
9191
hive:
9292
container_name: hive
93-
image: 'aiurtech/hive:0.8.1'
93+
image: 'aiurtech/hive:0.8.2'
9494
networks:
9595
privnet:
9696
ipv4_address: 172.15.0.11
9797
ports:
9898
- 10001:10001
9999
hostname: hive
100-
cpus: '2'
100+
# cpus: '2'
101101
mem_limit: 1000m
102102
mem_reservation: 300m
103103
depends_on:
@@ -107,7 +107,7 @@ services:
107107
- spark-master
108108
spark-master:
109109
container_name: spark-master
110-
image: 'aiurtech/spark-master:0.8.1'
110+
image: 'aiurtech/spark-master:0.8.2'
111111
networks:
112112
privnet:
113113
ipv4_address: 172.15.0.30
@@ -116,7 +116,7 @@ services:
116116
- 7077:7077
117117
- 9090:9090
118118
hostname: spark-master
119-
cpus: '2'
119+
# cpus: '2'
120120
mem_limit: 600m
121121
mem_reservation: 300m
122122
depends_on:
@@ -126,14 +126,14 @@ services:
126126
- hive-metastore
127127
spark-historyserver:
128128
container_name: spark-historyserver
129-
image: 'aiurtech/spark-historyserver:0.8.1'
129+
image: 'aiurtech/spark-historyserver:0.8.2'
130130
networks:
131131
privnet:
132132
ipv4_address: 172.15.0.31
133133
ports:
134134
- 18080:18080
135135
hostname: spark-historyserver
136-
cpus: '0.50'
136+
# cpus: '0.50'
137137
mem_limit: 500m
138138
mem_reservation: 300m
139139
depends_on:
@@ -142,14 +142,14 @@ services:
142142
- spark-master
143143
spark-worker:
144144
container_name: spark-worker
145-
image: 'aiurtech/spark-worker:0.8.1'
145+
image: 'aiurtech/spark-worker:0.8.2'
146146
networks:
147147
privnet:
148148
ipv4_address: 172.15.0.40
149149
ports:
150150
- 8081:8081
151151
hostname: spark-worker
152-
cpus: '4'
152+
# cpus: '4'
153153
mem_limit: 2000m
154154
mem_reservation: 500m
155155
depends_on:

‎hadoop-base/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/virgo-base:0.8.1
1+
FROM aiurtech/virgo-base:0.8.2
22

33
RUN adduser --system virgo
44
RUN groupadd hadoop

‎hadoop-datanode/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.8.1
1+
FROM aiurtech/hadoop-base:0.8.2
22

33
ENV HADOOP_OPTS "-Ddfs.data.dir=/hadoop/datanode -Dfs.defaultFS=hdfs://hadoop-namenode:8020"
44

‎hadoop-namenode/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.8.1
1+
FROM aiurtech/hadoop-base:0.8.2
22

33
RUN mkdir -p /hadoop/namenode && chown -R virgo:hadoop /hadoop/namenode
44

‎hive-metastore/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.8.1
1+
FROM aiurtech/hadoop-base:0.8.2
22

33
ENV HIVE_HOME=$VIRGO_HOME/apache-hive-$HIVE_VERSION
44
ENV PATH $HIVE_HOME/bin:$PATH

‎hive/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hive-metastore:0.8.1
1+
FROM aiurtech/hive-metastore:0.8.2
22

33
EXPOSE 10001
44

‎hive/run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ touch /tmp/virgo/hive.log
1616
ln -sf /dev/stdout /tmp/virgo/hive.log
1717

1818
function validateHive() {
19-
psql -U hive -h hive-metastore-postgresql -d metastore -t -c 'select * from "VERSION"' 2> /dev/null | wc -l
19+
PGPASSWORD=hive psql -U hive -h hive-metastore-postgresql -d metastore -t -c 'select * from "VERSION"' 2> /dev/null | wc -l
2020
}
2121

2222
echo "Waiting for Hive metastore to initialize schema"

‎k8s/hadoop-datanode-deployment.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: hadoop-datanode
5+
labels:
6+
app: hadoop-datanode
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: hadoop-datanode
12+
template:
13+
metadata:
14+
labels:
15+
app: hadoop-datanode
16+
spec:
17+
containers:
18+
- name: hadoop-datanode
19+
image: aiurtech/hadoop-datanode:0.8.2
20+
ports:
21+
- containerPort: 50010
22+
- containerPort: 50020
23+
- containerPort: 50075
24+
resources:
25+
requests:
26+
memory: "300Mi"
27+
cpu: "500m"
28+
limits:
29+
memory: "500Mi"
30+
cpu: "2000m"
31+
hostname: hadoop-datanode
32+
restartPolicy: Always

‎k8s/hadoop-datanode-service.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: hadoop-datanode
5+
spec:
6+
selector:
7+
app: hadoop-datanode
8+
ports:
9+
- name: "dn-port"
10+
port: 50010
11+
targetPort: 50010
12+
- name: "dn-ipc-port"
13+
port: 50020
14+
targetPort: 50075
15+
- name: "dn-http-port"
16+
port: 50075
17+
targetPort: 50075

‎k8s/hadoop-namenode-deployment.yaml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: hadoop-namenode
5+
labels:
6+
app: hadoop-namenode
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: hadoop-namenode
12+
template:
13+
metadata:
14+
labels:
15+
app: hadoop-namenode
16+
spec:
17+
containers:
18+
- name: hadoop-namenode
19+
image: aiurtech/hadoop-namenode:0.8.2
20+
ports:
21+
- containerPort: 8020
22+
- containerPort: 50070
23+
resources:
24+
requests:
25+
memory: "300Mi"
26+
cpu: "250m"
27+
limits:
28+
memory: "500Mi"
29+
cpu: "500m"
30+
hostname: hadoop-namenode
31+
restartPolicy: Always

‎k8s/hadoop-namenode-service.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: hadoop-namenode
5+
spec:
6+
selector:
7+
app: hadoop-namenode
8+
ports:
9+
- name: "nn-port"
10+
port: 8020
11+
targetPort: 8020
12+
- name: "nn-http-port"
13+
port: 50070
14+
targetPort: 50070

‎k8s/hive-deployment.yaml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: hive
5+
labels:
6+
app: hive
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: hive
12+
template:
13+
metadata:
14+
labels:
15+
app: hive
16+
spec:
17+
containers:
18+
- name: hive
19+
image: aiurtech/hive:0.8.2
20+
ports:
21+
- containerPort: 10001
22+
resources:
23+
requests:
24+
memory: "256Mi"
25+
cpu: "500m"
26+
limits:
27+
memory: "600Mi"
28+
cpu: "1000m"
29+
hostname: hive
30+
restartPolicy: Always

‎k8s/hive-metastore-deployment.yaml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: hive-metastore
5+
labels:
6+
app: hive-metastore
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: hive-metastore
12+
template:
13+
metadata:
14+
labels:
15+
app: hive-metastore
16+
spec:
17+
containers:
18+
- name: hive-metastore
19+
image: aiurtech/hive-metastore:0.8.2
20+
ports:
21+
- containerPort: 9083
22+
resources:
23+
requests:
24+
memory: "256Mi"
25+
cpu: "500m"
26+
limits:
27+
memory: "600Mi"
28+
cpu: "1000m"
29+
hostname: hive-metastore
30+
restartPolicy: Always
31+

0 commit comments

Comments
 (0)