Changes in spark port and docker file

Pradusekar · Pradusekar · commit 75ab015c0475 · 2020-12-13T16:31:08.000+04:00
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -3,23 +3,27 @@ services:
   spark-master:
     image: spydernaz/spark-master:latest
     ports:
-      - "9090:8080"
+      - "8080:8888"
       - "7077:7077"
     volumes:
        - ./apps:/opt/spark-apps
        - ./data:/opt/spark-data
     environment:
       - "SPARK_LOCAL_IP=spark-master"
+      - "SPARK_PUBLIC_DNS=localhost"
   spark-worker:
     image: spydernaz/spark-worker:latest
     depends_on:
       - spark-master
+    ports:
+      - "8081:8889"
     environment:
       - SPARK_MASTER=spark://spark-master:7077
       - SPARK_WORKER_CORES=1
       - SPARK_WORKER_MEMORY=1G
       - SPARK_DRIVER_MEMORY=128m
       - SPARK_EXECUTOR_MEMORY=256m
+      - "SPARK_PUBLIC_DNS=localhost"
     volumes:
        - ./apps:/opt/spark-apps
        - ./data:/opt/spark-data
diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile
@@ -1,20 +1,19 @@
 FROM openjdk:8u212-b04-jdk-stretch
-LABEL author="Nathaniel Vala" email="nathanielvala@hotmail.com"
-LABEL version="0.2"
+LABEL author="Pradeep Sekar" email="pradipraniboy@gmail.com"
+LABEL version="0.1"
 
 ENV DAEMON_RUN=true
-ENV SPARK_VERSION=2.4.3
-ENV HADOOP_VERSION=2.7
-ENV SCALA_VERSION=2.12.4
+ENV SPARK_VERSION=3.0.1
+ENV HADOOP_VERSION=3.2
+ENV SCALA_VERSION=2.12.10
 ENV SCALA_HOME=/usr/share/scala
 ENV SPARK_HOME=/spark
 
-
+##  Get all certificates.
 RUN apt-get update && apt-get install -y curl vim wget software-properties-common ssh net-tools ca-certificates jq
 
-#     apt update && apt -y upgrade \
-#     apt install -y wget ca-certificates && \
-#     apt install -y curl bash jq && \
+
+##  Install all initial lirary 
 RUN cd "/tmp" && \
     wget --no-verbose "https://downloads.typesafe.com/scala/${SCALA_VERSION}/scala-${SCALA_VERSION}.tgz" && \
     tar xzf "scala-${SCALA_VERSION}.tgz" && \
@@ -24,34 +23,18 @@ RUN cd "/tmp" && \
     ln -s "${SCALA_HOME}/bin/"* "/usr/bin/" && \
     rm -rf "/tmp/"*
 
-
-# RUN apk add --no-cache --virtual=.build-dependencies wget ca-certificates && \
-#     apk add --no-cache bash curl jq && \
-#     cd "/tmp" && \
-#     wget --no-verbose "https://downloads.typesafe.com/scala/${SCALA_VERSION}/scala-${SCALA_VERSION}.tgz" && \
-#     tar xzf "scala-${SCALA_VERSION}.tgz" && \
-#     mkdir "${SCALA_HOME}" && \
-#     rm "/tmp/scala-${SCALA_VERSION}/bin/"*.bat && \
-#     mv "/tmp/scala-${SCALA_VERSION}/bin" "/tmp/scala-${SCALA_VERSION}/lib" "${SCALA_HOME}" && \
-#     ln -s "${SCALA_HOME}/bin/"* "/usr/bin/" && \
-#     apk del .build-dependencies && \
-#     rm -rf "/tmp/"*
-
-
 # Add Dependencies for PySpark
 RUN apt-get install -y python3 python3-pip python3-numpy python3-matplotlib python3-scipy python3-pandas python3-simpy
 RUN update-alternatives --install "/usr/bin/python" "python" "$(which python3)" 1
 
 
-#Scala instalation
+#Scala instalation ---> Check the spark and sbt versions
 RUN export PATH="/usr/local/sbt/bin:$PATH" &&  apt update && apt install ca-certificates wget tar && mkdir -p "/usr/local/sbt" && wget -qO - --no-check-certificate "https://github.com/sbt/sbt/releases/download/v1.2.8/sbt-1.2.8.tgz" | tar xz -C /usr/local/sbt --strip-components=1 && sbt sbtVersion
 
 RUN wget --no-verbose http://apache.mirror.iphh.net/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
       && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark \
       && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
 
-
-
 # Fix the value of PYTHONHASHSEED
 # Note: this is needed when you use Python 3.3 or greater
 ENV PYTHONHASHSEED 1
diff --git a/docker/spark-master/Dockerfile b/docker/spark-master/Dockerfile
@@ -2,6 +2,7 @@ FROM spydernaz/spark-base:latest
 
 COPY start-master.sh /
 
+ENV SPARK_MASTER_HOST 127.0.0.1
 ENV SPARK_MASTER_PORT 7077
 ENV SPARK_MASTER_WEBUI_PORT 8080
 ENV SPARK_MASTER_LOG /spark/logs
diff --git a/docker/spark-master/start-master.sh b/docker/spark-master/start-master.sh
@@ -1,15 +1,17 @@
 #!/bin/bash
 
-export SPARK_MASTER_HOST=`hostname`
-
 . "/spark/sbin/spark-config.sh"
 
 . "/spark/bin/load-spark-env.sh"
 
+export SPARK_MASTER_HOST=127.0.0.1
+
 mkdir -p $SPARK_MASTER_LOG
 
 export SPARK_HOME=/spark
 
 ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out
 
+echo $SPARK_MASTER_HOST
+
 cd /spark/bin && /spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master --ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out
