Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
1f23554
Added git notes ..
Zarquan Jan 21, 2020
d71ed2b
Added git notes ..
Zarquan Jan 21, 2020
2a4bc75
Ansible code to create gateway and master nodes
Zarquan Jan 22, 2020
ae7f50d
Gateway, masters and workers created
Zarquan Jan 23, 2020
d86c8ff
Network, gateway, masters and workers created
Zarquan Jan 23, 2020
8c83b5c
format-notes
Zarquan Jan 23, 2020
9d807ad
Notes on git branch rename
Zarquan Jan 23, 2020
b9e1d37
Notes on git branch rename
Zarquan Jan 23, 2020
03782f0
Installing Java ..
Zarquan Jan 24, 2020
f903117
Refactored the SSH config generator
Zarquan Jan 24, 2020
fd51761
Hadoop install and config
Zarquan Jan 25, 2020
ddb6114
Added disc space limit
Zarquan Jan 25, 2020
08f74e5
Spark install
Zarquan Jan 26, 2020
95b9edb
Notes on configuring and running Hadoop and Spark processes
Zarquan Jan 27, 2020
d22c488
SSH keys for Hadoop masters
Zarquan Jan 30, 2020
3939c4f
SSH config for Hadoop masters
Zarquan Jan 30, 2020
651aeb2
SSH config for Hadoop masters
Zarquan Jan 30, 2020
ed37cef
Refactored SSH and DNS generators
Zarquan Jan 31, 2020
3b3957d
SSH from master to worker
Zarquan Jan 31, 2020
5be33c9
Hadoop ports
Zarquan Feb 3, 2020
365767a
Typos and tweaks
Feb 16, 2020
7c9bc79
Typos and tweaks
Zarquan Feb 16, 2020
24ba7f5
Clean up script to delete resources created by Ansible
Zarquan Feb 16, 2020
eb44416
Notes ...
Zarquan Feb 16, 2020
30e0980
Ansible deployment ....
Zarquan Apr 3, 2020
e19194f
HDFS configuration
Zarquan Apr 6, 2020
46bd1cc
HDFS config
Zarquan Apr 7, 2020
5c93072
HDFS working
Zarquan Apr 7, 2020
b9238a4
format-notes
Zarquan Apr 7, 2020
edb2a56
HDFS and YARN config
Zarquan Apr 9, 2020
b92646b
Notes and fixes
Zarquan Apr 11, 2020
1ca85bd
....
Zarquan Apr 11, 2020
30bf834
Step down to Java 8, YARN working
Zarquan Apr 12, 2020
07983ba
....
Zarquan Apr 14, 2020
86b9080
Spark works
Zarquan Apr 14, 2020
807f639
....
Zarquan Apr 14, 2020
158fc96
PySpark works
Zarquan Apr 15, 2020
e94665d
Conversion to Parquet and upload to Swift
Zarquan Apr 21, 2020
4308a60
Manila shares ..
Zarquan Apr 21, 2020
fc0b308
Notes on ObjectStore file upload
Zarquan Apr 23, 2020
0d2eda1
Notes on creating Manila shares
Zarquan Apr 23, 2020
6257d70
Moving Swift and Manila storage to another branch
Zarquan Apr 23, 2020
649beeb
Removed formatting tags from notes
Zarquan Apr 23, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions experiments/zrq/ansible/01-create-network.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#
# <meta:header>
# <meta:licence>
# Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
#
# This information is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This information is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# </meta:licence>
# </meta:header>
#
#

- name: "Create our internal network"
gather_facts: false
hosts: localhost
vars_files:
- /tmp/ansible-vars.yml
tasks:

- name: "Create our internal network"
os_network:
cloud: "{{ cloudname }}"
state: present
name: "{{ networks['internal'] }}"
shared: 'no'
external: 'no'
register:
agnetwork


- name: "Create a subnet for our internal network"
os_subnet:
cloud: "{{ cloudname }}"
state: present
name: "{{ agnetwork.network.name }}-subnet"
network_name: "{{ agnetwork.network.name }}"
cidr: 10.10.0.0/16
register:
agsubnet


- name: "Create a router for our internal network"
os_router:
cloud: "{{ cloudname }}"
state: present
name: "{{ agnetwork.network.name }}-router"
network: "{{ networks['external'] }}"
interfaces:
- net: "{{ agnetwork.network.name }}"
subnet: "{{ agsubnet.subnet.name }}"
register:
agrouter


Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#
# <meta:header>
# <meta:licence>
# Copyright (c) 2019, ROE (http://www.roe.ac.uk/)
# Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
#
# This information is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
Expand All @@ -20,81 +20,77 @@
#
#

- name: Create our gateway node
- name: "Create our gateway node"
gather_facts: false
hosts: localhost

vars_files:
- /tmp/ansible-vars.yml
tasks:

- name: "Create the '{{ firewalls['gateway'] }}' security group"
- name: "Create a security group for our gateway node"
os_security_group:
cloud: cumulus
cloud: "{{ cloudname }}"
state: present
name: "{{ firewalls['gateway'] }}"
name: "{{ security['gateway'] }}"
register:
security
gatewaysec

- name: Create a security rule for IPv4 SSH
- name: "Add a security rule for IPv4 SSH"
os_security_group_rule:
cloud: cumulus
cloud: "{{ cloudname }}"
state: present
security_group: "{{ security.id }}"
security_group: "{{ gatewaysec.id }}"
direction: 'ingress'
protocol: 'tcp'
ethertype: 'IPv4'
port_range_min: 22
port_range_max: 22
remote_ip_prefix: '0.0.0.0/0'

- name: Create a security rule for IPv6 SSH
- name: "Add a security rule for IPv6 SSH"
os_security_group_rule:
cloud: cumulus
cloud: "{{ cloudname }}"
state: present
security_group: "{{ security.id }}"
security_group: "{{ gatewaysec.id }}"
direction: 'ingress'
protocol: 'tcp'
ethertype: 'IPv6'
port_range_min: 22
port_range_max: 22
remote_ip_prefix: '::/0'

- name: Create our gateway node
- name: "Create our gateway node"
os_server:
cloud: 'cumulus'
cloud: "{{ cloudname }}"
state: present
name: "{{ hostvars['gateway'].nodename }}"
image: "{{ hostvars['gateway'].image }}"
flavor: "{{ hostvars['gateway'].flavor }}"
network: 'cumulus-internal'
key_name: 'aglais-ansible'
name: "{{ buildtag }}-gateway"
image: "{{ hostvars['gateway'].image }}"
flavor: "{{ hostvars['gateway'].flavor }}"
network: "{{ networks['internal'] }}"
key_name: "{{ keypair }}"
auto_ip: no
meta:
hostname: "{{ hostvars['gateway'].nodename }}"
hostname: 'gateway'
security_groups:
- "{{ security.id }}"
- "{{ gatewaysec.id }}"
register:
instance
gatewaynode

- name: Assign a new floating IP address
- name: "Assign a floating IP address to our gateway node"
os_floating_ip:
cloud: 'cumulus'
cloud: "{{ cloudname }}"
state: present
server: '{{ instance.id }}'
network: 'internet'
nat_destination: 'cumulus-internal'
server: '{{ gatewaynode.id }}'
network: "{{ networks['external'] }}"
nat_destination: "{{ networks['internal'] }}"
register:
floating
gatewayip

- name: Apply our security group to the floating IP address
- name: "Apply the gateway security group to our floating IP address"
os_port:
cloud: "{{ cloudname }}"
state: present
name: '{{ floating.floating_ip.port }}'
name: '{{ gatewayip.floating_ip.port }}'
security_groups:
- "{{ security.id }}"

- debug:
var: floating.floating_ip.floating_ip_address

- set_fact:


- "{{ gatewaysec.id }}"

98 changes: 0 additions & 98 deletions experiments/zrq/ansible/02-create-worker-nodes.yml

This file was deleted.

67 changes: 67 additions & 0 deletions experiments/zrq/ansible/03-create-masters.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#
# <meta:header>
# <meta:licence>
# Copyright (c) 2020, ROE (http://www.roe.ac.uk/)
#
# This information is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This information is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# </meta:licence>
# </meta:header>
#
#

- name: "Create our master nodes"
gather_facts: false
hosts: localhost
vars_files:
- /tmp/ansible-vars.yml
tasks:

- name: "Create a security group for our masters"
os_security_group:
cloud: "{{ cloudname }}"
state: present
name: "{{ security['masters'] }}"
register:
mastersec

- name: "Add a rule to allow SSH from our gateway"
os_security_group_rule:
cloud: "{{ cloudname }}"
state: present
security_group: "{{ mastersec.id }}"
direction: 'ingress'
protocol: 'tcp'
port_range_min: 22
port_range_max: 22
remote_group: "{{ security['gateway'] }}"

- name: "Create our masters"
os_server:
cloud: "{{ cloudname }}"
state: present
name: "{{ buildtag }}-{{ item }}"
image: "{{ hostvars[item]['image'] }}"
flavor: "{{ hostvars[item]['flavor'] }}"
network: "{{ networks['internal'] }}"
key_name: "{{ keypair }}"
auto_ip: no
meta:
"hostname={{ item }}"
security_groups:
- "{{ mastersec.id }}"
loop:
"{{ groups['masters'] }}"
register:
masternodes

Loading