change

Author: nghoanglong

Dockerfile

@@ -3,14 +3,18 @@ FROM ubuntu:18.04
 WORKDIR /root
 
 RUN apt-get update && apt-get install -y \
-    python3-pip \
     openssh-server \
     nano \
-    openjdk-8-jdk \
-    python3.7
+    openjdk-8-jdk
 
-RUN pip3 install jupyter && \
-    pip3 install pyspark
+RUN apt-get install -y python3 && \ 
+    apt-get install -y python3-pip && \
+    ln -s /usr/bin/python3 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --upgrade setuptools && \
+    pip3 install pyspark && \
+    pip3 install jupyter
 
 # download hadoop
 RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz && \
@@ -19,10 +23,10 @@ RUN wget https://archive.apache.org/dist/hadoop/common/hadoop-2.7.7/hadoop-2.7.7
     rm hadoop-2.7.7.tar.gz
 
 # download spark
-RUN wget https://dlcdn.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz && \
-    tar -xzf spark-3.2.1-bin-hadoop2.7.tgz && \
-    mv spark-3.2.1-bin-hadoop2.7 /usr/local/spark && \
-    rm spark-3.2.1-bin-hadoop2.7.tgz
+RUN wget https://archive.apache.org/dist/spark/spark-3.2.4/spark-3.2.4-bin-hadoop2.7.tgz && \
+    tar -xzf spark-3.2.4-bin-hadoop2.7.tgz && \
+    mv spark-3.2.4-bin-hadoop2.7 /usr/local/spark && \
+    rm spark-3.2.4-bin-hadoop2.7.tgz
 
 # set environment vars
 ENV HADOOP_HOME=/usr/local/hadoop
@@ -40,7 +44,7 @@ RUN ssh-keygen -t rsa -f ~/.ssh/id_rsa -P '' && \
     chmod 0600 ~/.ssh/authorized_keys
 
 # copy hadoop configs
-COPY config/* /tmp/
+COPY /spark_configs/* /tmp/
 
 RUN mv /tmp/ssh_config ~/.ssh/config && \
     mv /tmp/hadoop-env.sh $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \

build-image.sh

@@ -1,37 +1,53 @@
 version: '3'
 
 services:
-  hadoop-spark-master:
-    container_name: hadoop-spark-master
-    image: ghcr.io/nghoanglong/spark-cluster-with-docker/spark-cluster:1.0
+  hadoop-namenode:
+    container_name: hadoop-namenode
+    build:
+      context: .
+      dockerfile: Dockerfile
     networks:
       - hadoop-spark
+    mem_limit: 2g
+    cpus: 1
     ports:
       - "50070:50070"
       - "8088:8088"
       - "8080:8080"
       - "7077:7077"
       - "8888:8888"
       - "18080:18080"
-    hostname: hadoop-spark-master
+      - "4040:4040"
+      - "10000:10000"
+    hostname: hadoop-namenode
     command: ["sh", "-c", "service ssh start; ./start-cluster.sh; tail -f /dev/null"]
-  hadoop-spark-slave1:
-    container_name: hadoop-spark-slave1
-    image: ghcr.io/nghoanglong/spark-cluster-with-docker/spark-cluster:1.0
+  hadoop-datanode1:
+    container_name: hadoop-datanode1
+    build:
+      context: .
+      dockerfile: Dockerfile
+    mem_limit: 1.5g
+    cpus: 2
+    ports:
+      - "8081:8081"
     networks:
       - hadoop-spark
-    hostname: hadoop-spark-slave1
+    hostname: hadoop-datanode1
     depends_on:
-      - hadoop-spark-master
+      - hadoop-namenode
     command: ["sh", "-c", "service ssh start; tail -f /dev/null"]
-  hadoop-spark-slave2:
-    container_name: hadoop-spark-slave2
-    image: ghcr.io/nghoanglong/spark-cluster-with-docker/spark-cluster:1.0
+  hadoop-datanode2:
+    container_name: hadoop-datanode2
+    build:
+      context: .
+      dockerfile: Dockerfile
     networks:
       - hadoop-spark
-    hostname: hadoop-spark-slave2
+    mem_limit: 1.5g
+    cpus: 2
+    hostname: hadoop-datanode2
     depends_on:
-      - hadoop-spark-master
+      - hadoop-namenode
     command: ["sh", "-c", "service ssh start; tail -f /dev/null"]
 
 networks:

config/core-site.xml

@@ -0,0 +1,10 @@
+127.0.0.1	localhost
+172.18.0.2  hadoop-namenode
+172.18.0.3  hadoop-datanode2
+172.18.0.4  hadoop-datanode1
+
+::1	localhost ip6-localhost ip6-loopback
+fe00::0	ip6-localnet
+ff00::0	ip6-mcastprefix
+ff02::1	ip6-allnodes
+ff02::2	ip6-allrouters

config/hdfs-site.xml

@@ -0,0 +1,10 @@
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://hadoop-namenode:9000/</value>
+    </property>
+    <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/home/${user.name}/hadoop</value>
+    </property>
+</configuration>

config/slaves

@@ -0,0 +1,10 @@
+<configuration>
+    <property>
+        <name>dfs.namenode.heartbeat.recheck-interval</name>
+        <value>3000</value>
+    </property>
+    <property>
+        <name>dfs.replication</name>
+        <value>2</value>
+    </property>
+</configuration>

config/spark-default.conf

@@ -0,0 +1,2 @@
+hadoop-datanode1
+hadoop-datanode2

config/workers

@@ -0,0 +1,12 @@
+spark.master    yarn
+spark.driver.memory     2G
+spark.yarn.am.memory    1G
+spark.executor.memory	1G
+spark.executor.cores	1
+spark.history.ui.port 18080
+
+spark.ui.enabled true
+spark.ui.port 4040
+spark.eventLog.enabled true
+spark.eventLog.dir file:///tmp/spark-events
+spark.history.fs.logDirectory file:///tmp/spark-events

docker-compose.yaml

@@ -40,8 +40,6 @@
 # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1).
 # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G)
 # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G)
-export SPARK_DRIVER_MEMORY=2G
-export SPARK_EXECUTOR_MEMORY=1G
 
 # Options for the daemons used in the standalone deploy mode
 # - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname
@@ -68,6 +66,6 @@ export SPARK_EXECUTOR_MEMORY=1G
 # - SPARK_NO_DAEMONIZE  Run the proposed command in the foreground. It will not output a PID file.
 
 export SPARK_HOME=/usr/local/spark
-export PYSPARK_PYTHON=python3.7
+export PYSPARK_PYTHON=python3
 export PYSPARK_DRIVER_PYTHON=jupyter
 export PYSPARK_DRIVER_PYTHON_OPTS='notebook'

host_configs/hosts

@@ -4,6 +4,6 @@ Host localhost
 Host 0.0.0.0
   StrictHostKeyChecking no
 
-Host hadoop-spark-*
+Host hadoop-*
    StrictHostKeyChecking no
    UserKnownHostsFile=/dev/null

spark_configs/core-site.xml

@@ -0,0 +1,2 @@
+hadoop-datanode1
+hadoop-datanode2

config/hadoop-env.sh renamed to spark_configs/hadoop-env.sh

@@ -5,6 +5,6 @@
     </property>
     <property>
         <name>yarn.resourcemanager.hostname</name>
-        <value>hadoop-spark-master</value>
+        <value>hadoop-namenode</value>
     </property>
 </configuration>