From 6431b3a959cdef9be5dd47394784ee94d7258742 Mon Sep 17 00:00:00 2001 From: coderfi Date: Wed, 19 Feb 2014 23:54:22 -0800 Subject: [PATCH 1/6] - added Debian Wheezy specific installation scripts - now configures using the eth1 ip address (instead of eth0), this makes it easier to install when using a Vagrant vm - moved hosts to hosts.sample, updated README.md to specify customizing this file - moved group_vars/all to group_vars/all.sample, updated README.md to specify customizing this file - added boostrap.sh and site.sh safety checks to make sure the sample files were created - added other safety checks in case things are run twice --- .gitignore | 2 ++ README.md | 6 +++++- bootstrap/.gitignore | 2 ++ bootstrap/ansible_rsa.pub | 1 - bootstrap/bootstrap.sh | 13 ++++++++++++- bootstrap/hosts | 7 ------- group_vars/.gitignore | 1 + group_vars/{all => all.sample} | 3 ++- hosts => hosts.sample | 2 +- roles/cdh_hadoop_config/tasks/main.yml | 2 +- .../templates/hadoop-metrics2.properties | 14 +++++++------- roles/cdh_hadoop_config/templates/hdfs-site.xml | 2 +- roles/cdh_hadoop_config/templates/slaves | 2 +- roles/cdh_hbase_config/templates/regionservers | 2 +- roles/common/tasks/main.yml | 5 ++++- roles/common/templates/hosts | 4 +++- roles/ganglia_metad/templates/gmetad.conf | 2 +- roles/ganglia_monitor/templates/gmond.conf | 4 ++-- .../files/install_debian_webupd8team_repo.sh | 15 +++++++++++++++ roles/oracle_jdk/tasks/main.yml | 11 +++++++++-- roles/presto_common/tasks/main.yml | 4 ++-- roles/smokeping/templates/Targets | 4 ++-- .../td_agent/files/install_debian_libssl0.9.8.sh | 9 +++++++++ roles/td_agent/tasks/main.yml | 8 +++++++- roles/td_agent/templates/td-agent.conf | 2 +- site.sh | 14 +++++++++++++- site.yml | 4 ++-- 27 files changed, 106 insertions(+), 39 deletions(-) create mode 100644 .gitignore create mode 100644 bootstrap/.gitignore delete mode 100644 bootstrap/ansible_rsa.pub delete mode 100644 bootstrap/hosts create mode 100644 group_vars/.gitignore rename group_vars/{all => all.sample} (89%) rename hosts => hosts.sample (97%) create mode 100755 roles/oracle_jdk/files/install_debian_webupd8team_repo.sh create mode 100755 roles/td_agent/files/install_debian_libssl0.9.8.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a6acaa --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +hosts +.DS_Store diff --git a/README.md b/README.md index df75635..e0ad09d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ If you're assembling your own Hadoop playbook, these roles are available for you ### Configuration +Copy [`group_vars/all.sample`](group_vars/all.sample) to [`group_vars/all`](group_vars/all). + Set the following variables using `--extra-vars` or editing [`group_vars/all`](group_vars/all): Required: @@ -64,6 +66,8 @@ Optional: #### Adding hosts +Copy the [hosts.sample](hosts.sample) file to (hosts)[hosts]. + Edit the [hosts](hosts) file and list hosts per group (see [Inventory](http://www.ansibleworks.com/docs/intro_inventory.html) for more examples): ``` @@ -166,4 +170,4 @@ official [CDH4 Installation Guide](http://www.cloudera.com/content/cloudera-cont Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). -Copyright 2013-2014 [Mathias Bogaert](mailto:mathias.bogaert@gmail.com). \ No newline at end of file +Copyright 2013-2014 [Mathias Bogaert](mailto:mathias.bogaert@gmail.com). diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore new file mode 100644 index 0000000..6f8079c --- /dev/null +++ b/bootstrap/.gitignore @@ -0,0 +1,2 @@ +ansible_rsa.pub +hosts.sample diff --git a/bootstrap/ansible_rsa.pub b/bootstrap/ansible_rsa.pub deleted file mode 100644 index aab0499..0000000 --- a/bootstrap/ansible_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -paste your public SSH RSA key here \ No newline at end of file diff --git a/bootstrap/bootstrap.sh b/bootstrap/bootstrap.sh index 35523e1..70d7f08 100755 --- a/bootstrap/bootstrap.sh +++ b/bootstrap/bootstrap.sh @@ -1,3 +1,14 @@ #!/bin/bash + +if [ ! -f "ansible_rsa.pub" ]; then + echo "Please create a ansible_rsa.pub file with your ssh key." + exit +fi + +if [ ! -f "hosts" ]; then + echo "Please create a hosts inventory file (see hosts.sample)." + exit +fi + export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -i hosts -k -K -u ansibler bootstrap.yml \ No newline at end of file +ansible-playbook -i hosts -k -K -u ansibler -vvvv bootstrap.yml diff --git a/bootstrap/hosts b/bootstrap/hosts deleted file mode 100644 index 693415c..0000000 --- a/bootstrap/hosts +++ /dev/null @@ -1,7 +0,0 @@ -monitor01 -hmaster01 -hmaster02 -hslave01 -hslave02 -hslave03 -hslave04 \ No newline at end of file diff --git a/group_vars/.gitignore b/group_vars/.gitignore new file mode 100644 index 0000000..0702cb5 --- /dev/null +++ b/group_vars/.gitignore @@ -0,0 +1 @@ +all diff --git a/group_vars/all b/group_vars/all.sample similarity index 89% rename from group_vars/all rename to group_vars/all.sample index 56fbd4a..2c8ddaa 100644 --- a/group_vars/all +++ b/group_vars/all.sample @@ -8,4 +8,5 @@ mtu: 9216 # notify_email: notifications@example.com # postfix_domain: example.com # mandrill_username: your_username -# mandrill_api_key: your_api_key \ No newline at end of file +# mandrill_api_key: your_api_key + diff --git a/hosts b/hosts.sample similarity index 97% rename from hosts rename to hosts.sample index 695f472..9c15586 100644 --- a/hosts +++ b/hosts.sample @@ -79,4 +79,4 @@ journalnodes datanodes [2_links_aggregated:children] -2_link_aggregation_balance_alb \ No newline at end of file +2_link_aggregation_balance_alb diff --git a/roles/cdh_hadoop_config/tasks/main.yml b/roles/cdh_hadoop_config/tasks/main.yml index d47b408..0723a96 100644 --- a/roles/cdh_hadoop_config/tasks/main.yml +++ b/roles/cdh_hadoop_config/tasks/main.yml @@ -33,4 +33,4 @@ - name: run 'update-alternatives' to set hadoop configuration shell: update-alternatives --set hadoop-conf /etc/hadoop/conf.{{ site_name|lower }} - tags: hadoop \ No newline at end of file + tags: hadoop diff --git a/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties b/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties index 6deec94..6cbd71a 100644 --- a/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties +++ b/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties @@ -35,22 +35,22 @@ *.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 # journalnodes -journalnode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +journalnode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # namenodes -namenode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +namenode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # datanodes -datanode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +datanode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # jobtrackers -jobtracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +jobtracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # tasktrackers -tasktracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +tasktracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # maptasks -maptask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +maptask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # reducetasks -reducetask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +reducetask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} diff --git a/roles/cdh_hadoop_config/templates/hdfs-site.xml b/roles/cdh_hadoop_config/templates/hdfs-site.xml index de71ccf..8af92cc 100644 --- a/roles/cdh_hadoop_config/templates/hdfs-site.xml +++ b/roles/cdh_hadoop_config/templates/hdfs-site.xml @@ -133,4 +133,4 @@ dfs.client.file-block-storage-locations.timeout 3000 - \ No newline at end of file + diff --git a/roles/cdh_hadoop_config/templates/slaves b/roles/cdh_hadoop_config/templates/slaves index b4a6b6b..543ae9c 100644 --- a/roles/cdh_hadoop_config/templates/slaves +++ b/roles/cdh_hadoop_config/templates/slaves @@ -1,2 +1,2 @@ -{% for host in groups['datanodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }} +{% for host in groups['datanodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} {% endfor %} diff --git a/roles/cdh_hbase_config/templates/regionservers b/roles/cdh_hbase_config/templates/regionservers index a43d4cf..7aeff8c 100644 --- a/roles/cdh_hbase_config/templates/regionservers +++ b/roles/cdh_hbase_config/templates/regionservers @@ -1,2 +1,2 @@ -{% for host in groups['regionservers'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }} +{% for host in groups['regionservers'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} {% endfor %} \ No newline at end of file diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index ef07072..6577108 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -50,6 +50,9 @@ - name: create the hosts file for all machines template: backup=yes src=hosts dest=/etc/hosts +- name: create landscape dir + file: dest=/etc/landscape state=directory owner=root group=root mode=0644 + - name: configure landscape-sysinfo to hide link template: src=client.conf dest=/etc/landscape/client.conf owner=root group=root mode=0644 @@ -67,4 +70,4 @@ - name: make sure NTP is running and enabled service: name=ntp state=running enabled=yes - tags: ntp \ No newline at end of file + tags: ntp diff --git a/roles/common/templates/hosts b/roles/common/templates/hosts index f430f46..017ff7d 100644 --- a/roles/common/templates/hosts +++ b/roles/common/templates/hosts @@ -3,5 +3,7 @@ 127.0.0.1 localhost {% for host in groups['all'] %} -{{ hostvars[host]["ansible_default_ipv4"]["address"] }} {{ hostvars[host]["ansible_fqdn"] }} {{ hostvars[host]["ansible_hostname"] }} +# {{ host }} +{{ hostvars[host]["ansible_eth1"]["ipv4"]["address"] }} {{ hostvars[host]["ansible_fqdn"] }} {{ hostvars[host]["ansible_hostname"] }} + {% endfor %} diff --git a/roles/ganglia_metad/templates/gmetad.conf b/roles/ganglia_metad/templates/gmetad.conf index 0268666..ce957e7 100644 --- a/roles/ganglia_metad/templates/gmetad.conf +++ b/roles/ganglia_metad/templates/gmetad.conf @@ -40,7 +40,7 @@ # data_source "my grid" 50 1.3.4.7:8655 grid.org:8651 grid-backup.org:8651 # data_source "another source" 1.3.4.7:8655 1.3.4.8 -data_source "Hadoop" {{ hostvars[groups["masters"][0]]["ansible_default_ipv4"]["address"] }} {{ hostvars[groups["masters"][1]]["ansible_default_ipv4"]["address"] }} +data_source "Hadoop" {{ hostvars[groups["masters"][0]]['ansible_eth1']['ipv4']["address"] }} {{ hostvars[groups["masters"][1]]['ansible_eth1']['ipv4']["address"] }} # # Round-Robin Archives diff --git a/roles/ganglia_monitor/templates/gmond.conf b/roles/ganglia_monitor/templates/gmond.conf index 4af6abd..2b95639 100644 --- a/roles/ganglia_monitor/templates/gmond.conf +++ b/roles/ganglia_monitor/templates/gmond.conf @@ -36,12 +36,12 @@ host { also have each node send the same information to more than one place for redundancy. */ udp_send_channel { - host = {{ hostvars[groups["masters"][0]]["ansible_default_ipv4"]["address"] }} + host = {{ hostvars[groups["masters"][0]]['ansible_eth1']['ipv4']["address"] }} port = {{ ganglia_udp_send|default('8649') }} } udp_send_channel { - host = {{ hostvars[groups["masters"][1]]["ansible_default_ipv4"]["address"] }} + host = {{ hostvars[groups["masters"][1]]['ansible_eth1']['ipv4']["address"] }} port = {{ ganglia_udp_accept|default('8649') }} } diff --git a/roles/oracle_jdk/files/install_debian_webupd8team_repo.sh b/roles/oracle_jdk/files/install_debian_webupd8team_repo.sh new file mode 100755 index 0000000..463f042 --- /dev/null +++ b/roles/oracle_jdk/files/install_debian_webupd8team_repo.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +#see #see http://docs.ansible.com/script_module.html + +grep -Fqx "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" /etc/apt/sources.list + +if [ $? != 0 ]; then + echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list + echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 + apt-get update +fi #else already installed + + + diff --git a/roles/oracle_jdk/tasks/main.yml b/roles/oracle_jdk/tasks/main.yml index f827762..b8ff8cc 100644 --- a/roles/oracle_jdk/tasks/main.yml +++ b/roles/oracle_jdk/tasks/main.yml @@ -1,8 +1,15 @@ --- # file: roles/oracle_jdk/tasks/main.yml -- name: add webupd8team java repository +- name: add webupd8team java repository for ubuntu apt_repository: repo='ppa:webupd8team/java' + when: ansible_os_family == "Ubuntu" + +- name: add webupd8team java repository for debian + script: install_debian_webupd8team_repo.sh + when: + - ansible_os_family == "Debian" + - ansible_lsb["codename"] == "wheezy" - name: autoaccept license for java shell: executable=/bin/bash debconf-set-selections <<< "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" @@ -19,4 +26,4 @@ shell: creates=/usr/lib/jvm/floatingdecimal-0.1.jar chdir=/usr/lib/jvm curl -O http://repo1.maven.org/maven2/io/airlift/floatingdecimal/0.1/floatingdecimal-0.1.jar - name: set fact jdk_installed - set_fact: jdk_installed=true \ No newline at end of file + set_fact: jdk_installed=true diff --git a/roles/presto_common/tasks/main.yml b/roles/presto_common/tasks/main.yml index 001338e..bacaeed 100644 --- a/roles/presto_common/tasks/main.yml +++ b/roles/presto_common/tasks/main.yml @@ -14,7 +14,7 @@ tags: presto - name: install Presto at /usr/lib/presto - shell: chdir=/tmp curl http://central.maven.org/maven2/com/facebook/presto/presto-server/0.60/presto-server-0.60.tar.gz | tar xz && mv presto-server-0.60/* /usr/lib/presto + shell: chdir=/tmp creates=/usr/lib/presto/lib/presto-server-0.60.jar curl http://central.maven.org/maven2/com/facebook/presto/presto-server/0.60/presto-server-0.60.tar.gz | tar xz && mv presto-server-0.60/* /usr/lib/presto tags: presto - name: configure presto in /usr/lib/presto/etc @@ -38,4 +38,4 @@ - name: install upstart config for presto template: src=upstart.conf dest=/etc/init/presto.conf owner=root group=root mode=0644 - tags: presto \ No newline at end of file + tags: presto diff --git a/roles/smokeping/templates/Targets b/roles/smokeping/templates/Targets index 2d395d5..7d6adba 100644 --- a/roles/smokeping/templates/Targets +++ b/roles/smokeping/templates/Targets @@ -19,7 +19,7 @@ title = Hadoop Namenodes {% for host in groups['namenodes'] %} +++ {{ hostvars[host]['ansible_hostname'] }} menu = {{ hostvars[host]['ansible_hostname'] }} -title = Namenode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_default_ipv4']['address'] }} +title = Namenode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} host = {{ hostvars[host]['ansible_fqdn'] }} alerts = bigloss,someloss,startloss,rttdetect,hostdown {% endfor %} @@ -31,7 +31,7 @@ title = Hadoop Datanodes {% for host in groups['datanodes'] %} +++ {{ hostvars[host]['ansible_hostname'] }} menu = {{ hostvars[host]['ansible_hostname'] }} -title = Datanode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_default_ipv4']['address'] }} +title = Datanode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} host = {{ hostvars[host]['ansible_fqdn'] }} alerts = bigloss,someloss,startloss,rttdetect,hostdown {% endfor %} diff --git a/roles/td_agent/files/install_debian_libssl0.9.8.sh b/roles/td_agent/files/install_debian_libssl0.9.8.sh new file mode 100755 index 0000000..4efdc1a --- /dev/null +++ b/roles/td_agent/files/install_debian_libssl0.9.8.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh + +dpkg --list libssl > /dev/null +if [ $? != 0 ]; then + cd /tmp + wget http://ftp.us.debian.org/debian/pool/main/o/openssl/libssl0.9.8_0.9.8o-4squeeze14_amd64.deb + dpkg -i libssl0.9.8_0.9.8o-4squeeze14_amd64.deb +fi #else I guess we installed it already + diff --git a/roles/td_agent/tasks/main.yml b/roles/td_agent/tasks/main.yml index ca9fb50..d80d188 100644 --- a/roles/td_agent/tasks/main.yml +++ b/roles/td_agent/tasks/main.yml @@ -5,6 +5,12 @@ apt_repository: repo='deb http://packages.treasure-data.com/precise/ precise contrib' tags: tdagent +- name: install libssl0.9.8 for debian wheezy + script: install_debian_libssl0.9.8.sh + when: + - ansible_os_family == "Debian" + - ansible_lsb["codename"] == "wheezy" + - name: install td-agent via apt apt: pkg=td-agent update_cache=yes force=yes tags: tdagent @@ -25,4 +31,4 @@ template: src=td-agent.conf dest=/etc/td-agent/td-agent.conf owner=root group=root mode=0644 tags: tdagent notify: - - restart td-agent \ No newline at end of file + - restart td-agent diff --git a/roles/td_agent/templates/td-agent.conf b/roles/td_agent/templates/td-agent.conf index 4500149..1ce4549 100644 --- a/roles/td_agent/templates/td-agent.conf +++ b/roles/td_agent/templates/td-agent.conf @@ -19,5 +19,5 @@ type elasticsearch logstash_format true - host {{ hostvars[groups["monitors"][0]]["ansible_default_ipv4"]["address"] }} + host {{ hostvars[groups["monitors"][0]]['ansible_eth1']['ipv4']["address"] }} \ No newline at end of file diff --git a/site.sh b/site.sh index 4e75c1e..2fea234 100755 --- a/site.sh +++ b/site.sh @@ -6,6 +6,18 @@ export ANSIBLE_ERROR_ON_UNDEFINED_VARS=True export ANSIBLE_HOST_KEY_CHECKING=False export ANSIBLE_SSH_ARGS="-o ForwardAgent=yes" +if [ ! -f "hosts" ]; then + echo "Please create a hosts inventory file (see hosts.sample)." + exit +fi + +if [ ! -f "group_vars/all" ]; then + echo "Please create a group_vars/all file (see group_vars/all.sample)." + exit +fi + + + if [ $# -gt 0 ] then echo 'Running ansible-playbook -i hosts site.yml --tags' "$*" @@ -13,4 +25,4 @@ then else echo 'Running ansible-playbook -i hosts site.yml' ansible-playbook -i hosts --extra-vars "accelerate=true" site.yml -fi \ No newline at end of file +fi diff --git a/site.yml b/site.yml index 2ea68fd..0b7f7bb 100644 --- a/site.yml +++ b/site.yml @@ -107,7 +107,7 @@ sudo_user: hdfs tasks: - name: wait for the first namenode to come online - wait_for: host={{ hostvars[groups['namenodes'][0]]['ansible_default_ipv4']['address'] }} port=50070 + wait_for: host={{ hostvars[groups['namenodes'][0]]['ansible_eth1']['ipv4']['address'] }} port=50070 tags: - hadoop - hbase @@ -312,4 +312,4 @@ user: ansibler sudo: true roles: - - presto_worker \ No newline at end of file + - presto_worker From ac203ab1f723681dab221aaacc4a5d6f25fd5106 Mon Sep 17 00:00:00 2001 From: coderfi Date: Wed, 26 Feb 2014 08:55:12 -0800 Subject: [PATCH 2/6] Took out hardcoded ['ansible_eth1']['ipv4] references and restored previous ['ansible_default_ipv4'] instead It's probably better to figure out how to tell ansible (or in my usage case, Vagrant) how to set ansible_default_ipv4 properly anyway. Fixed bootstrap/.gitignore, now ignores bootstrap/hosts. Added bootstrap/hosts.sample (see docs) --- bootstrap/.gitignore | 2 +- bootstrap/hosts.sample | 7 +++++++ .../templates/hadoop-metrics2.properties | 14 +++++++------- roles/cdh_hadoop_config/templates/slaves | 2 +- roles/cdh_hbase_config/templates/regionservers | 4 ++-- roles/common/templates/hosts | 2 +- roles/ganglia_metad/templates/gmetad.conf | 2 +- roles/ganglia_monitor/templates/gmond.conf | 4 ++-- roles/smokeping/templates/Targets | 4 ++-- roles/td_agent/templates/td-agent.conf | 4 ++-- site.yml | 2 +- 11 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 bootstrap/hosts.sample diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore index 6f8079c..0661c76 100644 --- a/bootstrap/.gitignore +++ b/bootstrap/.gitignore @@ -1,2 +1,2 @@ ansible_rsa.pub -hosts.sample +hosts diff --git a/bootstrap/hosts.sample b/bootstrap/hosts.sample new file mode 100644 index 0000000..693415c --- /dev/null +++ b/bootstrap/hosts.sample @@ -0,0 +1,7 @@ +monitor01 +hmaster01 +hmaster02 +hslave01 +hslave02 +hslave03 +hslave04 \ No newline at end of file diff --git a/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties b/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties index 6cbd71a..6deec94 100644 --- a/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties +++ b/roles/cdh_hadoop_config/templates/hadoop-metrics2.properties @@ -35,22 +35,22 @@ *.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 # journalnodes -journalnode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +journalnode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # namenodes -namenode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +namenode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # datanodes -datanode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +datanode.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # jobtrackers -jobtracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +jobtracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # tasktrackers -tasktracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +tasktracker.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # maptasks -maptask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +maptask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} # reducetasks -reducetask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} +reducetask.sink.ganglia.servers={% for host in groups['namenodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }}:8649{% if not loop.last %},{% endif %}{% endfor %} diff --git a/roles/cdh_hadoop_config/templates/slaves b/roles/cdh_hadoop_config/templates/slaves index 543ae9c..b4a6b6b 100644 --- a/roles/cdh_hadoop_config/templates/slaves +++ b/roles/cdh_hadoop_config/templates/slaves @@ -1,2 +1,2 @@ -{% for host in groups['datanodes'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} +{% for host in groups['datanodes'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }} {% endfor %} diff --git a/roles/cdh_hbase_config/templates/regionservers b/roles/cdh_hbase_config/templates/regionservers index 7aeff8c..62478c7 100644 --- a/roles/cdh_hbase_config/templates/regionservers +++ b/roles/cdh_hbase_config/templates/regionservers @@ -1,2 +1,2 @@ -{% for host in groups['regionservers'] %}{{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} -{% endfor %} \ No newline at end of file +{% for host in groups['regionservers'] %}{{ hostvars[host]['ansible_default_ipv4']['address'] }} +{% endfor %} diff --git a/roles/common/templates/hosts b/roles/common/templates/hosts index 017ff7d..74b1e9f 100644 --- a/roles/common/templates/hosts +++ b/roles/common/templates/hosts @@ -4,6 +4,6 @@ {% for host in groups['all'] %} # {{ host }} -{{ hostvars[host]["ansible_eth1"]["ipv4"]["address"] }} {{ hostvars[host]["ansible_fqdn"] }} {{ hostvars[host]["ansible_hostname"] }} +{{ hostvars[host]["ansible_default_ipv4"]["address"] }} {{ hostvars[host]["ansible_fqdn"] }} {{ hostvars[host]["ansible_hostname"] }} {% endfor %} diff --git a/roles/ganglia_metad/templates/gmetad.conf b/roles/ganglia_metad/templates/gmetad.conf index ce957e7..751ad7d 100644 --- a/roles/ganglia_metad/templates/gmetad.conf +++ b/roles/ganglia_metad/templates/gmetad.conf @@ -40,7 +40,7 @@ # data_source "my grid" 50 1.3.4.7:8655 grid.org:8651 grid-backup.org:8651 # data_source "another source" 1.3.4.7:8655 1.3.4.8 -data_source "Hadoop" {{ hostvars[groups["masters"][0]]['ansible_eth1']['ipv4']["address"] }} {{ hostvars[groups["masters"][1]]['ansible_eth1']['ipv4']["address"] }} +data_source "Hadoop" {{ hostvars[groups["masters"][0]]['ansible_default_ipv4']["address"] }} {{ hostvars[groups["masters"][1]]['ansible_default_ipv4']["address"] }} # # Round-Robin Archives diff --git a/roles/ganglia_monitor/templates/gmond.conf b/roles/ganglia_monitor/templates/gmond.conf index 2b95639..694553f 100644 --- a/roles/ganglia_monitor/templates/gmond.conf +++ b/roles/ganglia_monitor/templates/gmond.conf @@ -36,12 +36,12 @@ host { also have each node send the same information to more than one place for redundancy. */ udp_send_channel { - host = {{ hostvars[groups["masters"][0]]['ansible_eth1']['ipv4']["address"] }} + host = {{ hostvars[groups["masters"][0]]['ansible_default_ipv4']["address"] }} port = {{ ganglia_udp_send|default('8649') }} } udp_send_channel { - host = {{ hostvars[groups["masters"][1]]['ansible_eth1']['ipv4']["address"] }} + host = {{ hostvars[groups["masters"][1]]['ansible_default_ipv4']["address"] }} port = {{ ganglia_udp_accept|default('8649') }} } diff --git a/roles/smokeping/templates/Targets b/roles/smokeping/templates/Targets index 7d6adba..2d395d5 100644 --- a/roles/smokeping/templates/Targets +++ b/roles/smokeping/templates/Targets @@ -19,7 +19,7 @@ title = Hadoop Namenodes {% for host in groups['namenodes'] %} +++ {{ hostvars[host]['ansible_hostname'] }} menu = {{ hostvars[host]['ansible_hostname'] }} -title = Namenode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} +title = Namenode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_default_ipv4']['address'] }} host = {{ hostvars[host]['ansible_fqdn'] }} alerts = bigloss,someloss,startloss,rttdetect,hostdown {% endfor %} @@ -31,7 +31,7 @@ title = Hadoop Datanodes {% for host in groups['datanodes'] %} +++ {{ hostvars[host]['ansible_hostname'] }} menu = {{ hostvars[host]['ansible_hostname'] }} -title = Datanode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_eth1']['ipv4']['address'] }} +title = Datanode {{ hostvars[host]['ansible_hostname'] }} at {{ hostvars[host]['ansible_default_ipv4']['address'] }} host = {{ hostvars[host]['ansible_fqdn'] }} alerts = bigloss,someloss,startloss,rttdetect,hostdown {% endfor %} diff --git a/roles/td_agent/templates/td-agent.conf b/roles/td_agent/templates/td-agent.conf index 1ce4549..2dc9977 100644 --- a/roles/td_agent/templates/td-agent.conf +++ b/roles/td_agent/templates/td-agent.conf @@ -19,5 +19,5 @@ type elasticsearch logstash_format true - host {{ hostvars[groups["monitors"][0]]['ansible_eth1']['ipv4']["address"] }} - \ No newline at end of file + host {{ hostvars[groups["monitors"][0]]['ansible_default_ipv4']["address"] }} + diff --git a/site.yml b/site.yml index 0b7f7bb..0d5e615 100644 --- a/site.yml +++ b/site.yml @@ -107,7 +107,7 @@ sudo_user: hdfs tasks: - name: wait for the first namenode to come online - wait_for: host={{ hostvars[groups['namenodes'][0]]['ansible_eth1']['ipv4']['address'] }} port=50070 + wait_for: host={{ hostvars[groups['namenodes'][0]]['ansible_default_ipv4']['address'] }} port=50070 tags: - hadoop - hbase From ce375da171441051e4d5a3849a9086aa8dc50ebf Mon Sep 17 00:00:00 2001 From: coderfi Date: Wed, 26 Feb 2014 08:57:59 -0800 Subject: [PATCH 3/6] removed ultra verbose -vvvv setting, replaced with -vv --- bootstrap/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/bootstrap.sh b/bootstrap/bootstrap.sh index 70d7f08..ec88f21 100755 --- a/bootstrap/bootstrap.sh +++ b/bootstrap/bootstrap.sh @@ -11,4 +11,4 @@ if [ ! -f "hosts" ]; then fi export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -i hosts -k -K -u ansibler -vvvv bootstrap.yml +ansible-playbook -i hosts -k -K -u ansibler -vv bootstrap.yml From ad2bf7a2008fc8a177f90f6d5060409c8d2f96f3 Mon Sep 17 00:00:00 2001 From: coderfi Date: Wed, 26 Feb 2014 09:00:27 -0800 Subject: [PATCH 4/6] consistent use of single and double quotes --- roles/ganglia_metad/templates/gmetad.conf | 2 +- roles/ganglia_monitor/templates/gmond.conf | 4 ++-- roles/td_agent/templates/td-agent.conf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/ganglia_metad/templates/gmetad.conf b/roles/ganglia_metad/templates/gmetad.conf index 751ad7d..0268666 100644 --- a/roles/ganglia_metad/templates/gmetad.conf +++ b/roles/ganglia_metad/templates/gmetad.conf @@ -40,7 +40,7 @@ # data_source "my grid" 50 1.3.4.7:8655 grid.org:8651 grid-backup.org:8651 # data_source "another source" 1.3.4.7:8655 1.3.4.8 -data_source "Hadoop" {{ hostvars[groups["masters"][0]]['ansible_default_ipv4']["address"] }} {{ hostvars[groups["masters"][1]]['ansible_default_ipv4']["address"] }} +data_source "Hadoop" {{ hostvars[groups["masters"][0]]["ansible_default_ipv4"]["address"] }} {{ hostvars[groups["masters"][1]]["ansible_default_ipv4"]["address"] }} # # Round-Robin Archives diff --git a/roles/ganglia_monitor/templates/gmond.conf b/roles/ganglia_monitor/templates/gmond.conf index 694553f..4af6abd 100644 --- a/roles/ganglia_monitor/templates/gmond.conf +++ b/roles/ganglia_monitor/templates/gmond.conf @@ -36,12 +36,12 @@ host { also have each node send the same information to more than one place for redundancy. */ udp_send_channel { - host = {{ hostvars[groups["masters"][0]]['ansible_default_ipv4']["address"] }} + host = {{ hostvars[groups["masters"][0]]["ansible_default_ipv4"]["address"] }} port = {{ ganglia_udp_send|default('8649') }} } udp_send_channel { - host = {{ hostvars[groups["masters"][1]]['ansible_default_ipv4']["address"] }} + host = {{ hostvars[groups["masters"][1]]["ansible_default_ipv4"]["address"] }} port = {{ ganglia_udp_accept|default('8649') }} } diff --git a/roles/td_agent/templates/td-agent.conf b/roles/td_agent/templates/td-agent.conf index 2dc9977..af7c578 100644 --- a/roles/td_agent/templates/td-agent.conf +++ b/roles/td_agent/templates/td-agent.conf @@ -19,5 +19,5 @@ type elasticsearch logstash_format true - host {{ hostvars[groups["monitors"][0]]['ansible_default_ipv4']["address"] }} + host {{ hostvars[groups["monitors"][0]]["ansible_default_ipv4"]["address"] }} From 063c22470797287167019dd52516319ed323899b Mon Sep 17 00:00:00 2001 From: fi Date: Sat, 25 Oct 2014 11:38:16 -0700 Subject: [PATCH 5/6] put back hosts (per pull request comments) added some documentation entries --- .gitignore | 1 - README.md | 3 ++- bootstrap/.gitignore | 1 - bootstrap/hosts | 7 +++++++ 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 bootstrap/hosts diff --git a/.gitignore b/.gitignore index 2a6acaa..e43b0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -hosts .DS_Store diff --git a/README.md b/README.md index 82f5080..9ef9345 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,8 @@ After the installation, go here: - Presto Coordinator at [hmaster02:8081](http://hmaster02:8081) - Hadoop Job History at [hslave01:19888](http://hslave01:19888) - Hadoop Node Manager at [hslave01:8042]](http://hslave01:8042) - - ElasticSearch at (monitor01:9200)[http://monitor01:9200/] + - ElasticSearch at (monitor01:9200)[http://monitor01:9200] + - Hadoop Journal Node at [hslave01:8480](http://hslave01:8480) - Hive CLI at (hmaster02) - ssh hmaster02; hive - Also see [cloudera.com](http://www.cloudera.com/content/cloudera-content/cloudera-docs/CM4Ent/latest/Cloudera-Manager-Installation-Guide/cmig_ports_cdh4.html) - CDH4 Ports diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore index 0661c76..8c0c399 100644 --- a/bootstrap/.gitignore +++ b/bootstrap/.gitignore @@ -1,2 +1 @@ ansible_rsa.pub -hosts diff --git a/bootstrap/hosts b/bootstrap/hosts new file mode 100644 index 0000000..8a7c7de --- /dev/null +++ b/bootstrap/hosts @@ -0,0 +1,7 @@ +monitor01 +hmaster01 +hmaster02 +hslave01 +#hslave02 +#hslave03 +#hslave04 From dd343e5a2e9090c8cd471a4d3c76985c8505f8ec Mon Sep 17 00:00:00 2001 From: fi Date: Sat, 25 Oct 2014 11:39:19 -0700 Subject: [PATCH 6/6] added ansible_rsa.pub per pull request comment --- bootstrap/.gitignore | 1 - bootstrap/ansible_rsa.pub | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 bootstrap/ansible_rsa.pub diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore index 8c0c399..e69de29 100644 --- a/bootstrap/.gitignore +++ b/bootstrap/.gitignore @@ -1 +0,0 @@ -ansible_rsa.pub diff --git a/bootstrap/ansible_rsa.pub b/bootstrap/ansible_rsa.pub new file mode 100644 index 0000000..5baf335 --- /dev/null +++ b/bootstrap/ansible_rsa.pub @@ -0,0 +1,2 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2LacJKsct+9Px4cIj2srg+pxdqn/qNKA8JEioKdaynZ3Dz78RR9OXpcsv7iYA6lu+/MGv7QO71mVezM5lpJ07floAEbLvpcyCZ0OO9RkHEldlS1X4XkQJpemhH9ycRnF+GduGBwAtv89KD3W6h0KKsBABt0zvIv5E2zNKOCvZZAmESYZn0o0c1K93pnP3QqIjSJ9971Qnp8rQN4OhdHp2j0BDuEYCsng5HlK2Nqyzo1YcaUjZDeUsNunW5r/nHe1MT2NeO+Q7amxvsOSN9wjA9ufsCfdNUPOTUQRBDVyiTVzHNiAy6p2XvtYhZsJcJfWb58a46Fj7KwuZdBo9XewT ansible-generated +