Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 2c6ec62

Browse files
author
Giannis Mouchakis
committed
Merge branch '2.0.0-hadoop3.1.1-java8'
2 parents 627ee46 + e823e6f commit 2c6ec62

File tree

17 files changed

+267
-281
lines changed

17 files changed

+267
-281
lines changed

‎Makefile‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
DOCKER_NETWORK = docker-hadoop_default
2+
ENV_FILE = hadoop.env
3+
current_branch := $(shell git rev-parse --abbrev-ref HEAD)
4+
build:
5+
docker build -t bde2020/hadoop-base:$(current_branch) ./base
6+
docker build -t bde2020/hadoop-namenode:$(current_branch) ./namenode
7+
docker build -t bde2020/hadoop-datanode:$(current_branch) ./datanode
8+
docker build -t bde2020/hadoop-resourcemanager:$(current_branch) ./resourcemanager
9+
docker build -t bde2020/hadoop-nodemanager:$(current_branch) ./nodemanager
10+
docker build -t bde2020/hadoop-historyserver:$(current_branch) ./historyserver
11+
docker build -t bde2020/hadoop-submit:$(current_branch) ./submit
12+
13+
wordcount:
14+
docker build -t hadoop-wordcount ./submit
15+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
16+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.1/README.txt /input/
17+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
18+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
19+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output
20+
docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /input

‎README.md‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,12 @@
22

33
# Changes
44

5-
Version 1.1.0 introduces healthchecks for the containers.
5+
Version 2.0.0 introduces uses wait_for_it script for the cluster startup
66

77
# Hadoop Docker
88

99
## Supported Hadoop Versions
10-
* 2.7.1 with OpenJDK 7
11-
* 2.7.1 with OpenJDK 8
10+
See repository branches for supported hadoop versions
1211

1312
## Quick Start
1413

@@ -17,13 +16,23 @@ To deploy an example HDFS cluster, run:
1716
docker-compose up
1817
```
1918

19+
Run example wordcount job:
20+
```
21+
make wordcount
22+
```
23+
24+
Or deploy in swarm:
25+
```
26+
docker stack deploy -c docker-compose-v3.yml hadoop
27+
```
28+
2029
`docker-compose` creates a docker network that can be found by running `docker network list`, e.g. `dockerhadoop_default`.
2130

2231
Run `docker network inspect` on the network (e.g. `dockerhadoop_default`) to find the IP the hadoop interfaces are published on. Access these interfaces with the following URLs:
2332

24-
* Namenode: http://<dockerhadoop_IP_address>:50070/dfshealth.html#tab-overview
33+
* Namenode: http://<dockerhadoop_IP_address>:9870/dfshealth.html#tab-overview
2534
* History server: http://<dockerhadoop_IP_address>:8188/applicationhistory
26-
* Datanode: http://<dockerhadoop_IP_address>:50075/
35+
* Datanode: http://<dockerhadoop_IP_address>:9864/
2736
* Nodemanager: http://<dockerhadoop_IP_address>:8042/node
2837
* Resource manager: http://<dockerhadoop_IP_address>:8088/
2938

@@ -49,5 +58,6 @@ The available configurations are:
4958
* /etc/hadoop/yarn-site.xml YARN_CONF
5059
* /etc/hadoop/httpfs-site.xml HTTPFS_CONF
5160
* /etc/hadoop/kms-site.xml KMS_CONF
61+
* /etc/hadoop/mapred-site.xml MAPRED_CONF
5262

5363
If you need to extend some other configuration file, refer to base/entrypoint.sh bash script.

‎base/Dockerfile‎

Lines changed: 6 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,58 +3,25 @@ MAINTAINER Ivan Ermilov <[email protected]>
33

44
RUN echo "deb http://ftp.debian.org/debian jessie-backports main" >> /etc/apt/sources.list
55

6-
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-8-jdk
6+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -t jessie-backports -y --no-install-recommends openjdk-8-jdk
77
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
88

9-
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends net-tools curl
9+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends net-tools curl netcat
1010

11-
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys \
12-
07617D4968B34D8F13D56E20BE5AAA0BA210C095 \
13-
2CAC83124870D88586166115220F69801F27E622 \
14-
4B96409A098DBD511DF2BC18DBAF69BEA7239D59 \
15-
9DD955653083EFED6171256408458C39E964B5FF \
16-
B6B3F7EDA5BA7D1E827DE5180DFF492D8EE2F25C \
17-
6A67379BEFC1AE4D5595770A34005598B8F47547 \
18-
47660BC98BC433F01E5C90581209E7F13D0C92B9 \
19-
CE83449FDC6DACF9D24174DCD1F99F6EE3CD2163 \
20-
A11DF05DEA40DA19CE4B43C01214CF3F852ADB85 \
21-
686E5EDF04A4830554160910DF0F5BBC30CD0996 \
22-
5BAE7CB144D05AD1BB1C47C75C6CC6EFABE49180 \
23-
AF7610D2E378B33AB026D7574FB955854318F669 \
24-
6AE70A2A38F466A5D683F939255ADF56C36C5F0F \
25-
70F7AB3B62257ABFBD0618D79FDB12767CC7352A \
26-
842AAB2D0BC5415B4E19D429A342433A56D8D31A \
27-
1B5D384B734F368052862EB55E43CAB9AEC77EAF \
28-
785436A782586B71829C67A04169AA27ECB31663 \
29-
5E49DA09E2EC9950733A4FF48F1895E97869A2FB \
30-
A13B3869454536F1852C17D0477E02D33DD51430 \
31-
A6220FFCC86FE81CE5AAC880E3814B59E4E11856 \
32-
EFE2E7C571309FE00BEBA78D5E314EEF7340E1CB \
33-
EB34498A9261F343F09F60E0A9510905F0B000F0 \
34-
3442A6594268AC7B88F5C1D25104A731B021B57F \
35-
6E83C32562C909D289E6C3D98B25B9B71EFF7770 \
36-
E9216532BF11728C86A11E3132CF4BF4E72E74D3 \
37-
E8966520DA24E9642E119A5F13971DA39475BD5D \
38-
1D369094D4CFAC140E0EF05E992230B1EB8C6EFA \
39-
A312CE6A1FA98892CB2C44EBA79AB712DE5868E6 \
40-
0445B7BFC4515847C157ECD16BA72FF1C99785DE \
41-
B74F188889D159F3D7E64A7F348C6D7A0DCED714 \
42-
4A6AC5C675B6155682729C9E08D51A0A7501105C \
43-
8B44A05C308955D191956559A5CEE20A90348D47
11+
RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
4412

45-
RUN gpg --keyserver pool.sks-keyservers.net --recv-key C36C5F0F
13+
RUN gpg --import KEYS
4614

47-
ENV HADOOP_VERSION 2.7.1
15+
ENV HADOOP_VERSION 3.1.1
4816
ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
4917
RUN set -x \
5018
&& curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \
5119
&& curl -fSL "$HADOOP_URL.asc" -o /tmp/hadoop.tar.gz.asc \
5220
&& gpg --verify /tmp/hadoop.tar.gz.asc \
5321
&& tar -xvf /tmp/hadoop.tar.gz -C /opt/ \
5422
&& rm /tmp/hadoop.tar.gz*
55-
23+
5624
RUN ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop
57-
RUN cp /etc/hadoop/mapred-site.xml.template /etc/hadoop/mapred-site.xml
5825
RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
5926

6027
RUN mkdir /hadoop-data

‎base/entrypoint.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ configure /etc/hadoop/hdfs-site.xml hdfs HDFS_CONF
3636
configure /etc/hadoop/yarn-site.xml yarn YARN_CONF
3737
configure /etc/hadoop/httpfs-site.xml httpfs HTTPFS_CONF
3838
configure /etc/hadoop/kms-site.xml kms KMS_CONF
39+
configure /etc/hadoop/mapred-site.xml mapred MAPRED_CONF
3940

4041
if [ "$MULTIHOMED_NETWORK" = "1" ]; then
4142
echo "Configuring for multihomed network"
@@ -78,4 +79,39 @@ if [ -n "$GANGLIA_HOST" ]; then
7879
done > /etc/hadoop/hadoop-metrics2.properties
7980
fi
8081

82+
function wait_for_it()
83+
{
84+
local serviceport=$1
85+
local service=${serviceport%%:*}
86+
local port=${serviceport#*:}
87+
local retry_seconds=5
88+
local max_try=100
89+
let i=1
90+
91+
nc -z $service $port
92+
result=$?
93+
94+
until [ $result -eq 0 ]; do
95+
echo "[$i/$max_try] check for ${service}:${port}..."
96+
echo "[$i/$max_try] ${service}:${port} is not available yet"
97+
if (( $i == $max_try )); then
98+
echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/"
99+
exit 1
100+
fi
101+
102+
echo "[$i/$max_try] try in ${retry_seconds}s once again ..."
103+
let "i++"
104+
sleep $retry_seconds
105+
106+
nc -z $service $port
107+
result=$?
108+
done
109+
echo "[$i/$max_try] $service:${port} is available."
110+
}
111+
112+
for i in ${SERVICE_PRECONDITION[@]}
113+
do
114+
wait_for_it ${i}
115+
done
116+
81117
exec $@

‎datanode/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
FROM bde2020/hadoop-base:1.1.0-hadoop2.7.1-java8
1+
FROM bde2020/hadoop-base:2.0.0-hadoop3.1.1-java8
22
MAINTAINER Ivan Ermilov <[email protected]>
33

4-
HEALTHCHECK CMD curl -f http://localhost:50075/ || exit 1
4+
HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1
55

66
ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data
77
RUN mkdir -p /hadoop/dfs/data
@@ -10,6 +10,6 @@ VOLUME /hadoop/dfs/data
1010
ADD run.sh /run.sh
1111
RUN chmod a+x /run.sh
1212

13-
EXPOSE 50075
13+
EXPOSE 9864
1414

1515
CMD ["/run.sh"]

‎docker-compose-local.yml‎

Lines changed: 0 additions & 95 deletions
This file was deleted.

‎docker-compose-nginx.yml‎

Lines changed: 0 additions & 71 deletions
This file was deleted.

0 commit comments

Comments
 (0)