Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit df6ee45

Browse files
author
Min Zhou
committed
Add features can update/restart/stop/remove cdh5 hadoop services
1 parent 09a23b2 commit df6ee45

File tree

29 files changed

+982
-414
lines changed

29 files changed

+982
-414
lines changed

‎ansible.cfg

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎group_vars/all

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
site_name: acme
1+
site_name: tango
22

33
# IPv4 maximum transmission unit, 9216 is the maximum for Intel/Cisco hardware
44
mtu: 9216
@@ -11,4 +11,53 @@ mtu: 9216
1111
# mandrill_api_key: your_api_key
1212

1313
# Upgrade kernel to 3.13, much improved epoll performance
14-
upgrade_kernel: no
14+
upgrade_kernel: no
15+
16+
# core-site_xml
17+
fs_trash_interval: 1440
18+
io_file_buffer_size: 131072
19+
20+
# hdfs-site_xml
21+
dfs_permissions_superusergroup: hdfs
22+
dfs_permissions_enabled: 'true'
23+
dfs_replication: 3
24+
dfs_blocksize: 268435456
25+
dfs_namenode_handler_count: 256
26+
dfs_datanode_handler_count: 32
27+
dfs_datanode_max_transfer_threads: 4096
28+
dfs_datanode_balanced_space_threshold: 10737418240
29+
dfs_datanode_balanced_space_preference_fraction: 0.75
30+
dfs_checksum_type: CRC32C
31+
32+
dfs_journalnode_edits_dir: /disk3/hadoop/data/dfs/jn
33+
dfs_namenode_name_dir:
34+
- /disk1/hadoop/data/dfs/nn
35+
- /disk2/hadoop/data/dfs/nn
36+
dfs_datanode_data_dir:
37+
- /disk1/hadoop/data/dfs/dn
38+
- /disk2/hadoop/data/dfs/dn
39+
- /disk3/hadoop/data/dfs/dn
40+
- /disk4/hadoop/data/dfs/dn
41+
42+
# yarn-site_xml
43+
yarn_nodemanager_local_dirs:
44+
- /disk1/hadoop/data/yarn/local
45+
- /disk2/hadoop/data/yarn/local
46+
- /disk3/hadoop/data/yarn/local
47+
- /disk4/hadoop/data/yarn/local
48+
yarn_nodemanager_log_dirs:
49+
- /disk1/hadoop/data/yarn/logs
50+
- /disk2/hadoop/data/yarn/logs
51+
- /disk3/hadoop/data/yarn/logs
52+
- /disk4/hadoop/data/yarn/logs
53+
yarn_nodemanager_vmem_pmem_ratio: 10
54+
yarn_nodemanager_resource_memory_mb: 24576
55+
yarn_nodemanager_pmem_check_enabled: 'true'
56+
yarn_nodemanager_vmem_check_enabled: 'true'
57+
58+
# mapred-site_xml
59+
mapreduce_map_memory_mb: 4096
60+
mapreduce_reduce_memory_mb: 8192
61+
mapreduce_map_java_opts: '-Xmx3072m'
62+
mapreduce_reduce_java_opts: '-Xmx6144m'
63+
mapreduce_jobtracker_handler_count: 128

‎roles/cdh_common/tasks/main.yml

Lines changed: 19 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,28 @@
11
---
22
# file: roles/cdh_common/tasks/main.yml
33

4-
- name: install Google Snappy
5-
shell: creates=/usr/local/lib/libsnappy.so.1.2.0 chdir=/tmp curl https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz | tar xz && cd snappy-1.1.1 && CFLAGS="-funroll-loops -O2 -DNDEBUG" CXXFLAGS="-funroll-loops -O2 -DNDEBUG" ./configure && make && checkinstall -pkgname=snappy-1.1.1 --backup=no --deldoc=yes --fstrans=no --default
6-
notify:
7-
- ldconfig
8-
tags: hadoop
4+
# - name: install Google Snappy
5+
# shell: creates=/usr/local/lib/libsnappy.so.1.2.0 chdir=/tmp curl https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz | tar xz && cd snappy-1.1.1 && CFLAGS="-funroll-loops -O2 -DNDEBUG" CXXFLAGS="-funroll-loops -O2 -DNDEBUG" ./configure && make && checkinstall -pkgname=snappy-1.1.1 --backup=no --deldoc=yes --fstrans=no --default
6+
# notify:
7+
# - ldconfig
8+
# tags: hadoop
99

10-
- name: add Cloudera CDH4 repository
11-
apt_repository: repo='deb [arch=amd64] http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh precise-cdh4 contrib'
10+
- name: add Cloudera CDH5 repository
11+
apt_repository: repo='deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5 contrib'
1212
tags:
13-
- hadoop
14-
- hbase
15-
- hive
13+
- update-apt
1614

17-
- name: add Cloudera CDH4 GPL extras repository
18-
apt_repository: repo='deb [arch=amd64] http://archive.cloudera.com/gplextras/ubuntu/precise/amd64/gplextras precise-gplextras4 contrib'
15+
- name: add Cloudera CDH5 GPL extras repository
16+
apt_repository: repo='deb [arch=amd64] http://mirror.infra.cloudera.com/archive/gplextras5/ubuntu/precise/amd64/gplextras precise-gplextras5 contrib' state=absent
1917
tags:
20-
- hadoop
21-
- hbase
22-
- hive
18+
- update-apt
2319

24-
- name: add Cloudera CDH4 key
25-
apt_key: url=http://archive.cloudera.com/cdh4/ubuntu/precise/amd64/cdh/archive.key
20+
- name: add Cloudera CDH5 GPL extras repository
21+
apt_repository: repo='deb [arch=amd64] http://archive.cloudera.com/gplextras5/ubuntu/precise/amd64/gplextras precise-gplextras5 contrib'
2622
tags:
27-
- hadoop
28-
- hbase
29-
- hive
23+
- update-apt
24+
25+
- name: add Cloudera CDH5 key
26+
apt_key: url=http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key state=present
27+
tags:
28+
- update-apt

‎roles/cdh_hadoop_common/tasks/main.yml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
- name: install base Hadoop packages via apt
55
apt: name={{ item }} update_cache=yes
66
with_items:
7-
- hadoop
8-
- hadoop-hdfs
9-
- hadoop-lzo-cdh4
10-
- hadoop-yarn
7+
- hadoop-lzo
118
tags: hadoop
129

1310
- name: create the /data directory
@@ -16,4 +13,10 @@
1613

1714
- name: create the /data/dfs directory
1815
file: path=/data/dfs owner=hdfs group=hdfs state=directory
19-
tags: hadoop
16+
tags: hadoop
17+
18+
- name: remove base Hadoop packages via apt
19+
apt: name={{ item }} state=absent
20+
with_items:
21+
- hadoop-lzo
22+
tags: remove-hadoop

‎roles/cdh_hadoop_config/defaults/main.yml

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
---
22
# file: roles/cdh_hadoop_config/defaults/main.yml
33

4-
# The default block size for new files, in bytes - here 256 MB
5-
dfs_blocksize: 268435456
6-
7-
# Upper bound on the number of files that it will serve at any one time
8-
max_xcievers: 4096
9-
104
# The Hadoop DataNode heap size
115
heapsize: 12278

‎roles/cdh_hadoop_config/tasks/.#main.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[email protected]:1404945029

‎roles/cdh_hadoop_config/tasks/main.yml

Lines changed: 100 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,65 @@
22
# file: roles/cdh_hadoop_config/tasks/main.yml
33

44
- name: copy /etc/hadoop/conf.empty to /etc/hadoop/conf.{{ site_name|lower }}
5-
command: creates=/etc/hadoop/conf.{{ site_name|lower }} cp -R -p /etc/hadoop/conf.empty /etc/hadoop/conf.{{ site_name|lower }}
5+
command: mkdir -m 755 -p /etc/hadoop/conf.{{ site_name|lower }}
66
tags:
7-
- hadoop
8-
- configuration
7+
- update-hadoop-config-dir
8+
- update-hadoop-config
9+
10+
- name: run 'update-alternatives' to install hadoop configuration
11+
command: update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.{{ site_name|lower }} 50
12+
tags:
13+
- update-hadoop-config-dir
14+
- update-hadoop-config
15+
16+
- name: run 'update-alternatives' to set hadoop configuration
17+
command: update-alternatives --set hadoop-conf /etc/hadoop/conf.{{ site_name|lower }}
18+
tags:
19+
- update-hadoop-config-dir
20+
- update-hadoop-config
21+
22+
# - name: configure hadoop in /etc/hadoop/conf.{{ site_name|lower }}
23+
# template: src={{ item }} dest=/etc/hadoop/conf.{{ site_name|lower }}/{{ item }} owner=root group=root mode=0644
24+
# with_items:
25+
# - core-site.xml
26+
# - hadoop-env.sh
27+
# - hadoop-metrics2.properties
28+
# - hdfs-site.xml
29+
# - org-xerial-snappy.properties
30+
# - slaves
31+
# - mapred-site.xml
32+
# - yarn-site.xml
33+
# - fair-scheduler.xml
34+
# notify:
35+
# - restart hadoop-hdfs-namenode
36+
# - restart hadoop-hdfs-journalnode
37+
# - restart hadoop-hdfs-datanode
38+
# - restart hadoop-mapreduce-historyserver
39+
# - restart hadoop-yarn-nodemanager
40+
# - restart hadoop-yarn-resourcemanager
41+
# tags:
42+
# - update-hadoop-configs
43+
944

1045
- name: configure hadoop in /etc/hadoop/conf.{{ site_name|lower }}
11-
template: src={{ item }} dest=/etc/hadoop/conf.{{ site_name|lower }}/{{ item }} owner=root group=root mode=0644
46+
template: src={{ item }} dest=/etc/hadoop/conf.{{ site_name|lower }}/{{ item }} owner=hadoop group=hadoop mode=0644
1247
with_items:
1348
- core-site.xml
49+
# notify:
50+
# - restart hadoop-hdfs-namenode
51+
# - restart hadoop-hdfs-journalnode
52+
# - restart hadoop-hdfs-datanode
53+
# - restart hadoop-mapreduce-historyserver
54+
# - restart hadoop-yarn-nodemanager
55+
# - restart hadoop-yarn-resourcemanager
56+
tags:
57+
- update-hadoop-core-site
58+
- update-hadoop-config
59+
60+
- name: configure hadoop in /etc/hadoop/conf.{{ site_name|lower }}
61+
template: src={{ item }} dest=/etc/hadoop/conf.{{ site_name|lower }}/{{ item }} owner=hadoop group=hadoop mode=0644
62+
with_items:
1463
- hadoop-env.sh
15-
- hadoop-metrics2.properties
16-
- hdfs-site.xml
17-
- org-xerial-snappy.properties
18-
- slaves
19-
- mapred-site.xml
20-
- yarn-site.xml
21-
- fair-scheduler.xml
2264
notify:
2365
- restart hadoop-hdfs-namenode
2466
- restart hadoop-hdfs-journalnode
@@ -27,21 +69,59 @@
2769
- restart hadoop-yarn-nodemanager
2870
- restart hadoop-yarn-resourcemanager
2971
tags:
72+
- update-hadoop-env
73+
- update-hadoop-config
3074
- hadoop
3175
- configuration
3276

33-
- name: update excluded datanodes
34-
copy: src=dfs.hosts.exclude dest=/etc/hadoop/conf/dfs.hosts.exclude owner=root group=root mode=644
77+
- name: configure hadoop in /etc/hadoop/conf.{{ site_name|lower }}
78+
template: src={{ item }} dest=/etc/hadoop/conf.{{ site_name|lower }}/{{ item }} owner=hdfs group=hdfs mode=0644
79+
with_items:
80+
- hdfs-site.xml
3581
notify:
36-
- refresh datanodes
82+
# - restart hadoop-hdfs-namenode
83+
# - restart hadoop-hdfs-datanode
3784
tags:
85+
- update-hadoop-hdfs-site
3886
- hadoop
3987
- configuration
4088

41-
- name: run 'update-alternatives' to install hadoop configuration
42-
command: update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.{{ site_name|lower }} 50
43-
tags: hadoop
89+
- name: update excluded datanodes
90+
copy: src=dfs.hosts.exclude dest=/etc/hadoop/conf/dfs.hosts.exclude owner=hdfs group=hdfs mode=644
91+
notify:
92+
- refresh datanodes
93+
tags:
94+
- update-hadoop-hdfs-excludes
4495

45-
- name: run 'update-alternatives' to set hadoop configuration
46-
command: update-alternatives --set hadoop-conf /etc/hadoop/conf.{{ site_name|lower }}
47-
tags: hadoop
96+
- name: configure yarn in /etc/hadoop/conf.{{ site_name|lower }}
97+
template: src={{ item }} dest=/etc/hadoop/conf/{{ item }} owner=yarn group=yarn mode=0644
98+
with_items:
99+
- yarn-site.xml
100+
# notify:
101+
# - restart hadoop-yarn-resourcemanager
102+
# - restart hadoop-yarn-nodemanager
103+
tags:
104+
- update-hadoop-yarn-site
105+
- update-hadoop-yarn-config
106+
107+
- name: configure yarn in /etc/hadoop/conf.{{ site_name|lower }}
108+
template: src={{ item }} dest=/etc/hadoop/conf/{{ item }} owner=yarn group=yarn mode=0644
109+
with_items:
110+
- fair-scheduler.xml
111+
# notify:
112+
# - restart hadoop-yarn-resourcemanager
113+
# - restart hadoop-yarn-nodemanager
114+
tags:
115+
- update-hadoop-yarn-scheduler
116+
- update-hadoop-yarn-config
117+
118+
- name: configure container executor in /etc/hadoop/conf.{{ site_name|lower }}
119+
template: src={{ item }} dest=/etc/hadoop/conf/{{ item }} owner=yarn group=yarn mode=0644
120+
with_items:
121+
- container-executor.cfg
122+
# notify:
123+
# - restart hadoop-yarn-resourcemanager
124+
# - restart hadoop-yarn-nodemanager
125+
tags:
126+
- update-hadoop-yarn-executor-config
127+
- update-hadoop-yarn-config

‎roles/cdh_hadoop_config/templates/container-executor.cfg

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# configured value of yarn.nodemanager.linux-container-executor.group
2+
yarn.nodemanager.linux-container-executor.group=yarn
3+
#comma separated list of users who can not run applications
4+
banned.users=hadoop,hdfs,zookeeper
5+
#Prevent other super-users
6+
min.user.id=1000
7+
##comma separated list of system users who CAN run applications
8+
allowed.system.users=

‎roles/cdh_hadoop_config/templates/core-site.xml

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,45 @@
55

66
<configuration>
77

8-
<property>
9-
<name>fs.defaultFS</name>
10-
<value>hdfs://{{ site_name|lower }}</value>
11-
<final>true</final>
12-
</property>
13-
14-
<property>
15-
<name>io.file.buffer.size</name>
16-
<value>131072</value>
17-
<final>true</final>
18-
</property>
19-
20-
<property>
21-
<name>io.compression.codecs</name>
22-
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
23-
<final>true</final>
24-
</property>
8+
<!-- the default path prefix used by the Hadoop FS client when none is given -->
9+
<property>
10+
<name>fs.defaultFS</name>
11+
<value>hdfs://{{ site_name|lower }}</value>
12+
</property>
13+
14+
<!-- The size of buffer for use in sequence files. -->
15+
<property>
16+
<name>io.file.buffer.size</name>
17+
<value>{{ io_file_buffer_size }}</value>
18+
<final>true</final>
19+
</property>
20+
21+
<!-- enable trash -->
22+
<property>
23+
<name>fs.trash.interval</name>
24+
<value>{{ fs_trash_interval }}</value>
25+
</property>
26+
27+
28+
<property>
29+
<name>io.compression.codecs</name>
30+
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
31+
<final>true</final>
32+
</property>
33+
34+
<property>
35+
<name>io.compression.codec.lzo.class</name>
36+
<value>com.hadoop.compression.lzo.LzoCodec</value>
37+
</property>
38+
39+
<!-- HTTPFS proxy user setting -->
40+
<!-- <property> -->
41+
<!-- <name>hadoop.proxyuser.httpfs.hosts</name> -->
42+
<!-- <value>*</value> -->
43+
<!-- </property> -->
44+
<!-- <property> -->
45+
<!-- <name>hadoop.proxyuser.httpfs.groups</name> -->
46+
<!-- <value>*</value> -->
47+
<!-- </property> -->
2548

2649
</configuration>

0 commit comments

Comments
 (0)