thành phần (centos 7)
192.168.8.200 master.hdfs.local
192.168.8.201 slave1.hdfs.local
192.168.8.202 slave2.hdfs.local
1. all server
(chạy với user root)
sudo adduser hadoop
passwd hadoop (setpassword: P@ssw0rd)
yum -y install java-1.8.0-openjdk
yum install java-1.8.0-openjdk-devel
2. master
su hadoop
ssh-keygen -t rsa
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
ssh-copy-id -i $HOME/.ssh/id_rsa.pub [email protected]
ssh-copy-id -i $HOME/.ssh/id_rsa.pub [email protected]
2.5 all server
su hadoop
chmod 0600 ~/.ssh/authorized_keys
3.all server
(root)
nano /etc/hosts
=> add this line
192.168.8.200 master.hdfs.local
192.168.8.201 slave1.hdfs.local
192.168.8.202 slave2.hdfs.local
---
(chạy với user hadoop)
wget https://dlcdn.apache.org/hadoop/common/hadoop-3.2.4/hadoop-3.2.4.tar.gz --no-
check-certificate
nano .bashrc
=> add line
#Hadoop Related Options
export HADOOP_HOME=$HOME/hadoop-3.2.4
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/nativ"
--
source ~/.bashrc
nano $HADOOP_HOME/etc/hadoop/hadoop-env.sh
=> fix this line: export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.392.b08-
2.el7_9.x86_64
(which javac -> readlink -f /usr/bin/javac -> /usr/lib/jvm/java-1.8.0-openjdk-
1.8.0.392.b08-2.el7_9.x86_64)
nano $HADOOP_HOME/etc/hadoop/core-site.xml
=> add line
<property>
<name>fs.defaultFS</name>
<value>hdfs://master.hdfs.local:9000</value>
</property>
4. master
nano $HADOOP_HOME/etc/hadoop/hdfs-site.xml
=> add line
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>192.168.8.200:50090</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/home/hadoop/hadoop-3.2.4/namenode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hadoop/hadoop-3.2.4/datanode</value>
</property>
5. slave
nano $HADOOP_HOME/etc/hadoop/hdfs-site.xml
=> add line
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>192.168.8.200:50090</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hadoop/hadoop-3.2.4/datanode</value>
</property>
6. all
nano $HADOOP_HOME/etc/hadoop/workers
=> add line
slave1.hdfs.local
slave2.hdfs.local
nano $HADOOP_HOME/etc/hadoop/mapred-site.xml
=> add line
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
nano $HADOOP_HOME/etc/hadoop/yarn-site.xml
=> add line
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
7. master
hdfs namenode -format
cd hadoop-3.2.4/sbin
./start-all.sh
jps
http://192.168.8.200:9870
LLL
https://phoenixnap.com/kb/install-hadoop-ubuntu
https://viblo.asia/p/cung-thiet-lap-multi-node-cluster-trong-hadoop-2x-nao-
5pPLkxXdVRZ
https://viblo.asia/p/hadoop-thi-co-lien-quan-gi-toi-big-data-gwd43MywLX9
https://viblo.asia/p/chua-phan-biet-duoc-database-data-warehouse-va-data-lake-
oK9VyQAqVQR
https://viblo.asia/p/pyspark-voi-mot-project-machine-learning-nho-nho-3RlL5GyB4bB
