diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e43b0f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/.travis.yml b/.travis.yml index 0a57763..b5ea337 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,15 @@ cache: directories: - $HOME/.pip-cache/ install: + - pip install python-keyczar==0.71c - pip install ansible --download-cache $HOME/.pip-cache - pip install dopy --download-cache $HOME/.pip-cache before_script: - - ansible-playbook -i hosts --extra-vars "api_key_password=$DO_API_KEY client_id=$DO_CLIENT_ID" do_cluster.yml + - ansible-playbook -i localhost --extra-vars "api_key_password=$DO_API_KEY client_id=$DO_CLIENT_ID" do_cluster.yml - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" - - "wget http://download.slimerjs.org/v0.9/0.9.1/slimerjs-0.9.1.zip" - - "unzip slimerjs-0.9.1.zip && mv slimerjs-0.9.1 ./slimerjs" + - "wget http://download.slimerjs.org/v0.9/0.9.0/slimerjs-0.9.0.zip" + - "unzip slimerjs-0.9.0.zip && mv slimerjs-0.9.0 ./slimerjs" - "gem install --version 0.8.9 faraday" - "gem install travis-artifacts" - eval `ssh-agent` @@ -21,14 +22,15 @@ before_script: - sleep 300 # DigitalOcean can be damn slow creating/booting droplets script: - ansible-playbook -i hosts --extra-vars "authorized_rsa_key='$(cat travis.ssh.pub)'" -u root bootstrap/bootstrap.yml - - ansible-playbook -i hosts site.yml + - rm -f /home/travis/.fireball.keys + - ansible-playbook -i hosts --extra-vars "accelerate=false" site.yml after_success: - "slimerjs screenshot.js http://hmaster01:50070 active-namenode.png" - "slimerjs screenshot.js http://hmaster02:50070 standby-namenode.png" - "slimerjs screenshot.js http://monitor01/ganglia ganglia.png" - "slimerjs screenshot.js http://monitor01:9200 elasticsearch.png" - "slimerjs screenshot.js http://monitor01/smokeping/smokeping.cgi?target=_charts smokeping.png" - - "slimerjs screenshot.js http://hmaster02:8081 presto.png" + - "slimerjs screenshot.js http://hmaster02:8080 presto.png" - "travis-artifacts upload --path active-namenode.png" - "travis-artifacts upload --path standby-namenode.png" - "travis-artifacts upload --path ganglia.png" diff --git a/README.md b/README.md index 4beb752..9ef9345 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ If you're assembling your own Hadoop playbook, these roles are available for you ### Configuration +Copy [`group_vars/all.sample`](group_vars/all.sample) to [`group_vars/all`](group_vars/all). + Set the following variables using `--extra-vars` or editing [`group_vars/all`](group_vars/all): Required: @@ -67,6 +69,8 @@ Optional: #### Adding hosts +Copy the [hosts.sample](hosts.sample) file to (hosts)[hosts]. + Edit the [hosts](hosts) file and list hosts per group (see [Inventory](http://www.ansibleworks.com/docs/intro_inventory.html) for more examples): ``` @@ -122,9 +126,15 @@ After the installation, go here: - Ganglia at [monitor01/ganglia](http://monitor01/ganglia/) - Kibana at [monitor01/kibana/index.html#/dashboard/file/logstash.json](http://monitor01/kibana/index.html#/dashboard/file/logstash.json) - Smokeping at [monitor01/smokeping/smokeping.cgi](http://monitor01/smokeping/smokeping.cgi) - - hmaster01 at [hmaster01:50070](http://hmaster01:50070) - active namenode - - hmaster02 at [hmaster02:50070](http://hmaster02:50070) - standby namenode - - Presto at [hmaster02:8081](http://hmaster02:8081) - Presto coordinator + - Hadoop Active Namenode at [hmaster01:50070](http://hmaster01:50070) + - Hadoop Standby Namenode at [hmaster02:50070](http://hmaster02:50070) + - Presto Coordinator at [hmaster02:8081](http://hmaster02:8081) + - Hadoop Job History at [hslave01:19888](http://hslave01:19888) + - Hadoop Node Manager at [hslave01:8042]](http://hslave01:8042) + - ElasticSearch at (monitor01:9200)[http://monitor01:9200] + - Hadoop Journal Node at [hslave01:8480](http://hslave01:8480) + - Hive CLI at (hmaster02) - ssh hmaster02; hive + - Also see [cloudera.com](http://www.cloudera.com/content/cloudera-content/cloudera-docs/CM4Ent/latest/Cloudera-Manager-Installation-Guide/cmig_ports_cdh4.html) - CDH4 Ports ### Performance testing @@ -170,4 +180,4 @@ official [CDH4 Installation Guide](http://www.cloudera.com/content/cloudera-cont Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). -Copyright 2013-2014 [Mathias Bogaert](mailto:mathias.bogaert@gmail.com). \ No newline at end of file +Copyright 2013-2014 [Mathias Bogaert](mailto:mathias.bogaert@gmail.com). diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/ansible_rsa.pub b/bootstrap/ansible_rsa.pub index aab0499..5baf335 100644 --- a/bootstrap/ansible_rsa.pub +++ b/bootstrap/ansible_rsa.pub @@ -1 +1,2 @@ -paste your public SSH RSA key here \ No newline at end of file +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2LacJKsct+9Px4cIj2srg+pxdqn/qNKA8JEioKdaynZ3Dz78RR9OXpcsv7iYA6lu+/MGv7QO71mVezM5lpJ07floAEbLvpcyCZ0OO9RkHEldlS1X4XkQJpemhH9ycRnF+GduGBwAtv89KD3W6h0KKsBABt0zvIv5E2zNKOCvZZAmESYZn0o0c1K93pnP3QqIjSJ9971Qnp8rQN4OhdHp2j0BDuEYCsng5HlK2Nqyzo1YcaUjZDeUsNunW5r/nHe1MT2NeO+Q7amxvsOSN9wjA9ufsCfdNUPOTUQRBDVyiTVzHNiAy6p2XvtYhZsJcJfWb58a46Fj7KwuZdBo9XewT ansible-generated + diff --git a/bootstrap/bootstrap.sh b/bootstrap/bootstrap.sh index 35523e1..08b84e0 100755 --- a/bootstrap/bootstrap.sh +++ b/bootstrap/bootstrap.sh @@ -1,3 +1,15 @@ #!/bin/bash + +if [ ! -f "ansible_rsa.pub" ]; then + echo "Please create a ansible_rsa.pub file with your ssh key." + exit +fi + +if [ ! -f "hosts" ]; then + echo "Please create a hosts inventory file (see hosts.sample)." +exit + +fi + export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -i hosts -k -K -u ansibler bootstrap.yml \ No newline at end of file +ansible-playbook -i hosts -k -K -u ansibler -vv bootstrap.yml diff --git a/bootstrap/hosts b/bootstrap/hosts index 693415c..8a7c7de 100644 --- a/bootstrap/hosts +++ b/bootstrap/hosts @@ -2,6 +2,6 @@ monitor01 hmaster01 hmaster02 hslave01 -hslave02 -hslave03 -hslave04 \ No newline at end of file +#hslave02 +#hslave03 +#hslave04 diff --git a/bootstrap/hosts.sample b/bootstrap/hosts.sample new file mode 100644 index 0000000..693415c --- /dev/null +++ b/bootstrap/hosts.sample @@ -0,0 +1,7 @@ +monitor01 +hmaster01 +hmaster02 +hslave01 +hslave02 +hslave03 +hslave04 \ No newline at end of file diff --git a/group_vars/.gitignore b/group_vars/.gitignore new file mode 100644 index 0000000..0702cb5 --- /dev/null +++ b/group_vars/.gitignore @@ -0,0 +1 @@ +all diff --git a/group_vars/all b/group_vars/all.sample similarity index 58% rename from group_vars/all rename to group_vars/all.sample index 5f36003..4c4486d 100644 --- a/group_vars/all +++ b/group_vars/all.sample @@ -11,4 +11,12 @@ mtu: 9216 # mandrill_api_key: your_api_key # Upgrade kernel to 3.13, much improved epoll performance -upgrade_kernel: no \ No newline at end of file +upgrade_kernel: no + +# replace the /etc/hosts file with the hosts and ip addresses of the cluster +muck_up_hosts: yes + +#use the IP address in this index +#0 is the first one, 1 is the second one, +#-2 is the second to last one, -1 is the last one +ansible_all_ipv4_addresses_index: 0 diff --git a/hosts b/hosts.sample similarity index 97% rename from hosts rename to hosts.sample index be609d4..a9883c7 100644 --- a/hosts +++ b/hosts.sample @@ -79,4 +79,4 @@ journalnodes datanodes [2_links_aggregated:children] -2_link_aggregation_balance_alb \ No newline at end of file +2_link_aggregation_balance_alb diff --git a/roles/apache2/tasks/main.yml b/roles/apache2/tasks/main.yml index e53cf6d..2e5a9c9 100644 --- a/roles/apache2/tasks/main.yml +++ b/roles/apache2/tasks/main.yml @@ -7,6 +7,9 @@ - apache2 tags: apache +- name: delete default site + file: dest=/etc/apache2/sites-enabled/000-default.conf state=absent + - name: configure apache2 so it doesn't complain 'can't determine fqdn' lineinfile: dest=/etc/apache2/apache2.conf regexp="{{ ansible_fqdn }}" line="ServerName {{ ansible_fqdn }}" notify: diff --git a/roles/cdh_hadoop_config/tasks/main.yml b/roles/cdh_hadoop_config/tasks/main.yml index 7aafb10..a76780f 100644 --- a/roles/cdh_hadoop_config/tasks/main.yml +++ b/roles/cdh_hadoop_config/tasks/main.yml @@ -44,4 +44,4 @@ - name: run 'update-alternatives' to set hadoop configuration command: update-alternatives --set hadoop-conf /etc/hadoop/conf.{{ site_name|lower }} - tags: hadoop \ No newline at end of file + tags: hadoop diff --git a/roles/cdh_hadoop_config/templates/hdfs-site.xml b/roles/cdh_hadoop_config/templates/hdfs-site.xml index a03f449..b525305 100644 --- a/roles/cdh_hadoop_config/templates/hdfs-site.xml +++ b/roles/cdh_hadoop_config/templates/hdfs-site.xml @@ -138,4 +138,4 @@ dfs.client.file-block-storage-locations.timeout 3000 - \ No newline at end of file + diff --git a/roles/cdh_hbase_config/templates/regionservers b/roles/cdh_hbase_config/templates/regionservers index 132642f..9433547 100644 --- a/roles/cdh_hbase_config/templates/regionservers +++ b/roles/cdh_hbase_config/templates/regionservers @@ -1,2 +1,2 @@ {% for host in groups['regionservers'] %}{{ hostvars[host].ipv4_address|default(hostvars[host].ansible_default_ipv4.address) }} -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index bb84164..7c003cf 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -51,8 +51,12 @@ - name: create the hosts file for all machines template: backup=yes src=hosts dest=/etc/hosts + when: muck_up_hosts tags: configuration +- name: create landscape dir + file: dest=/etc/landscape state=directory owner=root group=root mode=0644 + - name: configure landscape-sysinfo to hide link template: src=client.conf dest=/etc/landscape/client.conf owner=root group=root mode=0644 @@ -70,4 +74,4 @@ - name: make sure NTP is enabled and started service: name=ntp enabled=yes state=started - tags: ntp \ No newline at end of file + tags: ntp diff --git a/roles/oracle_jdk/files/install_debian_webupd8team_repo.sh b/roles/oracle_jdk/files/install_debian_webupd8team_repo.sh old mode 100644 new mode 100755 index e4e9495..84b3eee --- a/roles/oracle_jdk/files/install_debian_webupd8team_repo.sh +++ b/roles/oracle_jdk/files/install_debian_webupd8team_repo.sh @@ -9,4 +9,4 @@ if [ $? != 0 ]; then echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" | tee -a /etc/apt/sources.list apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 apt-get update -fi \ No newline at end of file +fi #else already installed diff --git a/roles/oracle_jdk/tasks/main.yml b/roles/oracle_jdk/tasks/main.yml index fba3d7c..5eb826d 100644 --- a/roles/oracle_jdk/tasks/main.yml +++ b/roles/oracle_jdk/tasks/main.yml @@ -30,4 +30,4 @@ - name: set fact jdk_installed set_fact: jdk_installed=true - tags: java \ No newline at end of file + tags: java diff --git a/roles/presto_common/tasks/main.yml b/roles/presto_common/tasks/main.yml index 110ad3c..4656060 100644 --- a/roles/presto_common/tasks/main.yml +++ b/roles/presto_common/tasks/main.yml @@ -1,6 +1,11 @@ --- # file: roles/presto_common/tasks/main.yml +- name: install upstart + apt: pkg=upstart state=present + when: + - ansible_distribution_release == "wheezy" + - name: create the /data/presto directory file: path=/data/presto state=directory tags: presto @@ -44,4 +49,4 @@ template: src=upstart.conf dest=/etc/init/presto.conf owner=root group=root mode=0644 tags: - configuration - - presto \ No newline at end of file + - presto diff --git a/roles/smokeping/tasks/main.yml b/roles/smokeping/tasks/main.yml index 824f2a4..edb1ab7 100644 --- a/roles/smokeping/tasks/main.yml +++ b/roles/smokeping/tasks/main.yml @@ -1,6 +1,8 @@ --- # file: roles/smokeping/tasks/main.yml +#FIXME: sendmail dependency needs to be met + - name: install smokeping via apt apt: pkg=smokeping tags: smokeping @@ -15,15 +17,32 @@ notify: - restart smokeping +#FIXME: Depends on Apache2 2.4 (Debian Wheezy is Apache2 2.2) +######### +# Debian Wheezy (Apache2 2.2) specific +#- name: link smokeping config in /etc/apache2/conf.d +# command: creates=/etc/apache2/conf.d/smokeping.conf chdir=/etc/apache2/conf.d ln -s ../../smokeping/apache2.conf smokeping.conf +# when: +# - ansible_distribution == "Debian" +# - ansible_distribution_release == "wheezy" +# tags: smokeping + +# Apache2 2.4 - name: link to smokeping config in /etc/apache2/conf-available command: creates=/etc/apache2/conf-available/smokeping.conf chdir=/etc/apache2/conf-available ln -s ../../smokeping/apache2.conf smokeping.conf - when: ansible_distribution_version not in ["12.04", "12.10", "13.04"] + when: + - ansible_distribution_version not in ["12.04", "12.10", "13.04"] +# - ansible_distribution_release != "wheezy" tags: smokeping - name: enable the smokeping apache2 configuration command: a2enconf smokeping - when: ansible_distribution_version not in ["12.04", "12.10", "13.04"] + when: + - ansible_distribution_version not in ["12.04", "12.10", "13.04"] +# - ansible_distribution_release != "wheezy" tags: smokeping +# +##################### - name: enable the cgid apache2 module command: a2enmod cgid diff --git a/roles/td_agent/files/install_debian_libssl0.9.8.sh b/roles/td_agent/files/install_debian_libssl0.9.8.sh old mode 100644 new mode 100755 index ee65b0b..0bbba38 --- a/roles/td_agent/files/install_debian_libssl0.9.8.sh +++ b/roles/td_agent/files/install_debian_libssl0.9.8.sh @@ -5,4 +5,4 @@ if [ $? != 0 ]; then cd /tmp wget http://ftp.us.debian.org/debian/pool/main/o/openssl/libssl0.9.8_0.9.8o-4squeeze14_amd64.deb dpkg -i libssl0.9.8_0.9.8o-4squeeze14_amd64.deb -fi \ No newline at end of file +fi #else I guess we installed it already diff --git a/roles/td_agent/tasks/main.yml b/roles/td_agent/tasks/main.yml index c823e9e..4ab0db1 100644 --- a/roles/td_agent/tasks/main.yml +++ b/roles/td_agent/tasks/main.yml @@ -44,4 +44,4 @@ - tdagent - configuration notify: - - restart td-agent \ No newline at end of file + - restart td-agent diff --git a/site.sh b/site.sh index aa9c61a..2fea234 100755 --- a/site.sh +++ b/site.sh @@ -4,12 +4,25 @@ IFS=$',' export ANSIBLE_ERROR_ON_UNDEFINED_VARS=True export ANSIBLE_HOST_KEY_CHECKING=False +export ANSIBLE_SSH_ARGS="-o ForwardAgent=yes" + +if [ ! -f "hosts" ]; then + echo "Please create a hosts inventory file (see hosts.sample)." + exit +fi + +if [ ! -f "group_vars/all" ]; then + echo "Please create a group_vars/all file (see group_vars/all.sample)." + exit +fi + + if [ $# -gt 0 ] then echo 'Running ansible-playbook -i hosts site.yml --tags' "$*" - ansible-playbook -i hosts site.yml --tags "$*" + ansible-playbook -i hosts --extra-vars "accelerate=true" site.yml --tags "$*" else echo 'Running ansible-playbook -i hosts site.yml' - ansible-playbook -i hosts site.yml -fi \ No newline at end of file + ansible-playbook -i hosts --extra-vars "accelerate=true" site.yml +fi diff --git a/site.yml b/site.yml index c2f6c83..1e6e820 100644 --- a/site.yml +++ b/site.yml @@ -6,7 +6,7 @@ user: ansibler tasks: - name: determine interface - set_fact: ipv4_address="{{ hostvars[inventory_hostname].ansible_default_ipv4.address }}" + set_fact: ipv4_address="{{ hostvars[inventory_hostname].ansible_all_ipv4_addresses[ansible_all_ipv4_addresses_index] }}" # eg. to use a eth1: ipv4_address="{{ hostvars[inventory_hostname].ansible_eth1.ipv4.address }}" - hosts: all @@ -45,9 +45,11 @@ user: ansibler sudo: true roles: - - td_agent + #needs libc6 2.14 :( + - { role: td_agent, when: ansible_distribution_release != "wheezy" } - rsyslog_fluentd + # Hadoop - hosts: zookeepers:journalnodes:resourcemanager:nodemanagers:historyserver:hbase_masters:regionservers