Skip to content

Commit 23982c2

Browse files
fernandoraccaFernando Racca Braskic
fernandoracca
and
Fernando Racca Braskic
authored
Upgraded PostgreSQL (#14)
Co-authored-by: Fernando Racca Braskic <[email protected]>
1 parent 6b1380d commit 23982c2

File tree

16 files changed

+31
-29
lines changed

16 files changed

+31
-29
lines changed

‎config.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env bash
22

33
#USER Config
4-
VERSION=0.7.5
4+
VERSION=0.8.1
55
NETWORK_NAME=virgo
66
NETWORK_SUBNET=172.15.0.0/16
77

‎docker-compose.yml

+11-11
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ version: '2.4'
33
services:
44
hadoop-namenode:
55
container_name: hadoop-namenode
6-
image: 'aiurtech/hadoop-namenode:0.7.5'
6+
image: 'aiurtech/hadoop-namenode:0.8.1'
77
networks:
88
privnet:
99
ipv4_address: 172.15.0.2
@@ -18,7 +18,7 @@ services:
1818
mem_reservation: 300m
1919
hadoop-datanode:
2020
container_name: hadoop-datanode
21-
image: 'aiurtech/hadoop-datanode:0.7.5'
21+
image: 'aiurtech/hadoop-datanode:0.8.1'
2222
networks:
2323
privnet:
2424
ipv4_address: 172.15.0.20
@@ -35,7 +35,7 @@ services:
3535
- hadoop-namenode
3636
yarn-resourcemanager:
3737
container_name: yarn-resourcemanager
38-
image: 'aiurtech/yarn-resourcemanager:0.7.5'
38+
image: 'aiurtech/yarn-resourcemanager:0.8.1'
3939
networks:
4040
privnet:
4141
ipv4_address: 172.15.0.3
@@ -51,20 +51,20 @@ services:
5151
mem_reservation: 300m
5252
yarn-nodemanager:
5353
container_name: yarn-nodemanager
54-
image: 'aiurtech/yarn-nodemanager:0.7.5'
54+
image: 'aiurtech/yarn-nodemanager:0.8.1'
5555
networks:
5656
privnet:
5757
ipv4_address: 172.15.0.4
5858
ports:
5959
- 8042:8042
6060
hostname: yarn-nodemanager
61-
mem_limit: 1200m
61+
mem_limit: 2400m
6262
mem_reservation: 300m
6363
depends_on:
6464
- yarn-resourcemanager
6565
hive-metastore-postgresql:
6666
container_name: hive-metastore-postgresql
67-
image: 'aiurtech/hive-metastore-postgresql:0.7.5'
67+
image: 'aiurtech/hive-metastore-postgresql:0.8.1'
6868
networks:
6969
privnet:
7070
ipv4_address: 172.15.0.5
@@ -76,7 +76,7 @@ services:
7676
mem_reservation: 300m
7777
hive-metastore:
7878
container_name: hive-metastore
79-
image: 'aiurtech/hive-metastore:0.7.5'
79+
image: 'aiurtech/hive-metastore:0.8.1'
8080
networks:
8181
privnet:
8282
ipv4_address: 172.15.0.10
@@ -90,7 +90,7 @@ services:
9090
- hive-metastore-postgresql
9191
hive:
9292
container_name: hive
93-
image: 'aiurtech/hive:0.7.5'
93+
image: 'aiurtech/hive:0.8.1'
9494
networks:
9595
privnet:
9696
ipv4_address: 172.15.0.11
@@ -107,7 +107,7 @@ services:
107107
- spark-master
108108
spark-master:
109109
container_name: spark-master
110-
image: 'aiurtech/spark-master:0.7.5'
110+
image: 'aiurtech/spark-master:0.8.1'
111111
networks:
112112
privnet:
113113
ipv4_address: 172.15.0.30
@@ -126,7 +126,7 @@ services:
126126
- hive-metastore
127127
spark-historyserver:
128128
container_name: spark-historyserver
129-
image: 'aiurtech/spark-historyserver:0.7.5'
129+
image: 'aiurtech/spark-historyserver:0.8.1'
130130
networks:
131131
privnet:
132132
ipv4_address: 172.15.0.31
@@ -142,7 +142,7 @@ services:
142142
- spark-master
143143
spark-worker:
144144
container_name: spark-worker
145-
image: 'aiurtech/spark-worker:0.7.5'
145+
image: 'aiurtech/spark-worker:0.8.1'
146146
networks:
147147
privnet:
148148
ipv4_address: 172.15.0.40

‎hadoop-base/Dockerfile

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/virgo-base:0.7.5
1+
FROM aiurtech/virgo-base:0.8.1
22

33
RUN adduser --system virgo
44
RUN groupadd hadoop
@@ -8,8 +8,7 @@ RUN chown virgo:hadoop ${VIRGO_HOME}
88

99
ENV HADOOP_HOME=$VIRGO_HOME/hadoop-$HADOOP_VERSION
1010
ENV HADOOP_CONF_DIR=/etc/hadoop
11-
ENV JVM_DOCKER_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1"
12-
ENV HADOOP_OPTS="${JVM_DOCKER_OPTS}"
11+
ENV HADOOP_OPTS="${VIRGO_JVM_DOCKER_OPTS}"
1312

1413
RUN mkdir -p ${HADOOP_CONF_DIR}
1514

‎hadoop-datanode/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.7.5
1+
FROM aiurtech/hadoop-base:0.8.1
22

33
ENV HADOOP_OPTS "-Ddfs.data.dir=/hadoop/datanode -Dfs.defaultFS=hdfs://hadoop-namenode:8020"
44

‎hadoop-namenode/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.7.5
1+
FROM aiurtech/hadoop-base:0.8.1
22

33
RUN mkdir -p /hadoop/namenode && chown -R virgo:hadoop /hadoop/namenode
44

‎hive-metastore-postgresql/Dockerfile

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
FROM postgres:9.5
1+
FROM postgres:11.7
22

33
LABEL maintainer "Aiur Tech <[email protected]>"
44

5+
ENV POSTGRES_PASSWORD airw5j6XtUCYc
6+
57
ADD init-hive-db.sh /docker-entrypoint-initdb.d/init-user-db.sh

‎hive-metastore/Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.7.5
1+
FROM aiurtech/hadoop-base:0.8.1
22

33
ENV HIVE_HOME=$VIRGO_HOME/apache-hive-$HIVE_VERSION
44
ENV PATH $HIVE_HOME/bin:$PATH
@@ -7,7 +7,7 @@ RUN set -x \
77
&& curl -fSL "https://www-eu.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz" -o /tmp/apache-hive-$HIVE_VERSION-bin.tar.gz \
88
&& tar -xf /tmp/apache-hive-$HIVE_VERSION-bin.tar.gz -C $VIRGO_HOME \
99
&& mv $VIRGO_HOME/apache-hive-$HIVE_VERSION-bin ${HIVE_HOME} \
10-
&& curl -fSL https://jdbc.postgresql.org/download/postgresql-9.4.1212.jar -o $HIVE_HOME/lib/postgresql-9.4.1212.jar \
10+
&& curl -fSL https://jdbc.postgresql.org/download/postgresql-42.2.13.jar -o $HIVE_HOME/lib/postgresql-9.4.1212.jar \
1111
&& chown -R virgo:hadoop $HIVE_HOME \
1212
&& rm /tmp/apache-hive-$HIVE_VERSION-bin.tar.gz*
1313

‎hive-metastore/run.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ hive --service metastore &
2323
sleep 10
2424

2525
function validateHive() {
26-
psql -U hive -h hive-metastore-postgresql -d metastore -t -c 'select * from "VERSION"' 2> /dev/null | wc -l
26+
PGPASSWORD=hive psql -U hive -h hive-metastore-postgresql -d metastore -t -c 'select * from "VERSION"' 2> /dev/null | wc -l
2727
}
2828

2929
echo "Waiting for PostgreSQL metastore to initialize schema"

‎hive/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hive-metastore:0.7.5
1+
FROM aiurtech/hive-metastore:0.8.1
22

33
EXPOSE 10001
44

‎spark-base/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.7.5
1+
FROM aiurtech/hadoop-base:0.8.1
22

33
ENV SPARK_HOME=$VIRGO_HOME/spark-$SPARK_VERSION
44

‎spark-historyserver/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/spark-base:0.7.5
1+
FROM aiurtech/spark-base:0.8.1
22

33
ENV SPARK_HISTORYSERVER_UI_PORT 18080
44

‎spark-master/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/spark-base:0.7.5
1+
FROM aiurtech/spark-base:0.8.1
22

33
ENV SPARK_MASTER_UI_PORT 9090
44

‎spark-worker/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/spark-base:0.7.5
1+
FROM aiurtech/spark-base:0.8.1
22

33
ENV SPARK_WORKER_UI_PORT 8081
44

‎virgo-base/Dockerfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@ FROM openjdk:8-jdk-slim-buster
22

33
LABEL maintainer "Aiur Tech <[email protected]>"
44

5-
ARG VIRGO_DIST_VERSION=0.7.5
5+
ARG VIRGO_DIST_VERSION=0.8.1
66
ENV VIRGO_VERSION ${VIRGO_DIST_VERSION}
77
ENV HADOOP_VERSION 2.7.7
88
ENV HIVE_VERSION 1.2.2
99
ENV SPARK_VERSION=2.3.0
1010
ENV VIRGO_HOME=/opt/virgo
11+
ENV VIRGO_JVM_DOCKER_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1"
1112

1213
RUN apt-get update \
1314
&& apt-get install --no-install-recommends --no-install-suggests -y vim netcat curl \

‎yarn-nodemanager/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.7.5
1+
FROM aiurtech/hadoop-base:0.8.1
22

33
COPY core-site.xml $HADOOP_CONF_DIR/core-site.xml
44
COPY hdfs-site.xml $HADOOP_CONF_DIR/hdfs-site.xml

‎yarn-resourcemanager/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM aiurtech/hadoop-base:0.7.5
1+
FROM aiurtech/hadoop-base:0.8.1
22

33
COPY yarn-site.xml $HADOOP_CONF_DIR/yarn-site.xml
44
COPY capacity-scheduler.xml $HADOOP_CONF_DIR/capacity-scheduler.xml

0 commit comments

Comments
 (0)