Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e1e1e21

Browse files
gmouchakisgmouchakis
authored
Merge pull request big-data-europe#52 from nevmmv/2.0.0-hadoop3.2.1-java8
moved to 3.2.1/stable hadoop version
2 parents bc861a6 + 1835c7b commit e1e1e21

File tree

17 files changed

+39
-33
lines changed

17 files changed

+39
-33
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ build:
1313
wordcount:
1414
docker build -t hadoop-wordcount ./submit
1515
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
16-
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/
16+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/
1717
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
1818
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
1919
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

‎base/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
1717

1818
RUN gpg --import KEYS
1919

20-
ENV HADOOP_VERSION 3.1.3
20+
ENV HADOOP_VERSION 3.2.1
2121
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
2222

2323
RUN set -x \
@@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
3333

3434
RUN mkdir /hadoop-data
3535

36-
ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION
36+
ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
3737
ENV HADOOP_CONF_DIR=/etc/hadoop
3838
ENV MULTIHOMED_NETWORK=1
3939
ENV USER=root
40-
ENV PATH $HADOOP_PREFIX/bin/:$PATH
40+
ENV PATH $HADOOP_HOME/bin/:$PATH
4141

4242
ADD entrypoint.sh /entrypoint.sh
4343

‎datanode/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM bde2020/hadoop-base
1+
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
22

33
MAINTAINER Ivan Ermilov <[email protected]>
44

‎datanode/run.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then
66
exit 2
77
fi
88

9-
$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
9+
$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode

‎docker-compose-v3.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '3'
22

33
services:
44
namenode:
5-
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
5+
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
66
networks:
77
- hbase
88
volumes:
@@ -24,7 +24,7 @@ services:
2424
traefik.port: 50070
2525

2626
datanode:
27-
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
27+
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
2828
networks:
2929
- hbase
3030
volumes:
@@ -42,7 +42,7 @@ services:
4242
traefik.port: 50075
4343

4444
resourcemanager:
45-
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
45+
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
4646
networks:
4747
- hbase
4848
environment:
@@ -64,7 +64,7 @@ services:
6464
disable: true
6565

6666
nodemanager:
67-
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
67+
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
6868
networks:
6969
- hbase
7070
environment:
@@ -80,7 +80,7 @@ services:
8080
traefik.port: 8042
8181

8282
historyserver:
83-
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
83+
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
8484
networks:
8585
- hbase
8686
volumes:

‎docker-compose.yml‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@ version: "3"
22

33
services:
44
namenode:
5-
image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
5+
image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
66
container_name: namenode
7+
restart: always
78
ports:
89
- 9870:9870
10+
- 9000:9000
911
volumes:
1012
- hadoop_namenode:/hadoop/dfs/name
1113
environment:
@@ -14,8 +16,9 @@ services:
1416
- ./hadoop.env
1517

1618
datanode:
17-
image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
19+
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
1820
container_name: datanode
21+
restart: always
1922
volumes:
2023
- hadoop_datanode:/hadoop/dfs/data
2124
environment:
@@ -24,26 +27,29 @@ services:
2427
- ./hadoop.env
2528

2629
resourcemanager:
27-
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
30+
image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
2831
container_name: resourcemanager
32+
restart: always
2933
environment:
30-
SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
34+
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
3135
env_file:
3236
- ./hadoop.env
3337

3438
nodemanager1:
35-
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
39+
image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
3640
container_name: nodemanager
41+
restart: always
3742
environment:
38-
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
43+
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
3944
env_file:
4045
- ./hadoop.env
4146

4247
historyserver:
43-
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
48+
image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
4449
container_name: historyserver
50+
restart: always
4551
environment:
46-
SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
52+
SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
4753
volumes:
4854
- hadoop_historyserver:/hadoop/yarn/timeline
4955
env_file:

‎hadoop.env‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
3838
MAPRED_CONF_mapreduce_reduce_memory_mb=8192
3939
MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
4040
MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
41-
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
42-
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
43-
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
41+
MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
42+
MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
43+
MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/

‎historyserver/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM bde2020/hadoop-base
1+
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
22

33
MAINTAINER Ivan Ermilov <[email protected]>
44

‎historyserver/run.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver
3+
$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver

‎namenode/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM bde2020/hadoop-base
1+
FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
22

33
MAINTAINER Ivan Ermilov <[email protected]>
44

0 commit comments

Comments
 (0)