Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Jul 5, 2018. It is now read-only.

Commit eae47fc

Browse files
uncleGenuncleGen
committed
add base work
1 parent ecb87ef commit eae47fc

File tree

21 files changed

+2085
-1
lines changed

21 files changed

+2085
-1
lines changed

‎README.md‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,18 @@
1-
# spark-on-ecs
1+
# Spark on ECS
2+
3+
## 介绍
4+
1. [quick start & manual](doc/manual.md)
5+
2. [打通SSH隧道](doc/ssh_tunnel.md)(可选)
6+
7+
## 特性
8+
--------
9+
1. 快速搭建基于ECS的Spark集群
10+
通过脚本一键式快速搭建高效、稳定的Spark集群运行环境，并可以通过命令行快捷、简单的管理集群的生命周期
11+
2. Spark工具支持
12+
运行环境集成Spark-notebook、Hue等，并开放原生的Spark UI，可以方便快捷的通过图形界面运行、调试代码和监控作业
13+
3. 安全
14+
所有Web页面的访问均采用SSH隧道技术，可以在非安全环境下使用端口转发来加密Web页面的访问，保护个人隐私以及重要商业信息
15+
4. 开放
16+
所有软件均使用社区开源版本，代码开放透明、文档丰富
17+
18+

‎bin/conf/nginx.conf.template‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
worker_processes 1;
2+
error_log logs/error.log;
3+
error_log logs/error.log notice;
4+
error_log logs/error.log info;
5+
pid logs/nginx.pid;
6+
7+
events {
8+
worker_connections 1024;
9+
}
10+
11+
http {
12+
include mime.types;
13+
default_type pplication/octet-stream;
14+
sendfile off;
15+
keepalive_timeout 20;
16+
gzip on;
17+
proxy_intercept_errors off;
18+
19+
${upstream_place_holder}
20+
21+
${server_place_holder}
22+
}

‎bin/core/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎bin/core/common.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/python
2+
#coding=utf-8
3+
import os
4+
import sys
5+
6+
class GlobalVar:
7+
8+
DEFAULT_CONF_DIR = "/root/.config"
9+
PROPERTY_FILE = "%s/packages.property" % DEFAULT_CONF_DIR
10+
HADOOP_INSTALL_DIR = "/opt/hadoop"
11+
HADOOP_CONF_DIR = "%s/etc/hadoop" % HADOOP_INSTALL_DIR
12+
SPARK_INSTALL_DIR = "/opt/spark"
13+
SPARK_CONF_DIR = "%s/conf" % SPARK_INSTALL_DIR
14+
SPARK_NOTEBOOK_INSTALL_DIR = "/opt/spark-notebook"
15+
HUE_INSTALL_DIR = "/opt/hue"
16+
ALIYUN_SDK_URL = "http://docs-aliyun-com-cn-b.oss-cn-hangzhou.aliyuncs.com/ecs/assets/sdk/python_sdk.tgz"
17+
SPARK_ECS_DIR = ""
18+
CLUSTER_STATUS = "%s/status/cluster-" % DEFAULT_CONF_DIR
19+
CLUSTER_INSTANCES = "%s/instances/" % DEFAULT_CONF_DIR
20+
CLUSTER_HOSTS = ""
21+
22+
ECS_API_PAGESIZE = 50
23+
24+
ECS_INSTANCE_TYPE = {
25+
"ecs.t1.small": (1, 1),
26+
"ecs.s1.small": (1, 2),
27+
"ecs.s1.medium": (1, 4),
28+
"ecs.s2.small": (2, 2),
29+
"ecs.s2.large": (2, 4),
30+
"ecs.s2.xlarge": (2, 8),
31+
"ecs.s3.medium": (4, 4),
32+
"ecs.s3.large": (4, 8),
33+
"ecs.m1.medium": (4, 16)
34+
}
35+
36+
ECS_REGION = {
37+
"1": "cn-hangzhou",
38+
"2": "cn-shenzhen",
39+
"3": "cn-beijing",
40+
"4": "cn-qingdao"
41+
}
42+
43+
SPARK_IMAGES = {
44+
("Spark-1.3.1", "cn-hangzhou"): "m-23xecoatf",
45+
("Spark-1.3.1", "cn-shenzhen"): "m-94ksoicp4",
46+
("Spark-1.3.1", "cn-beijing"): "m-25dt21m47",
47+
("Spark-1.3.1", "cn-qingdao"): "m-28w0wqwa6"
48+
}
49+
50+
AVAILABLE_SAPRK_VERSION = {
51+
"1": "Spark-1.3.1"
52+
}

‎bin/core/config_nginx.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/usr/bin/python
2+
#coding=utf-8
3+
import os
4+
import sys
5+
6+
def do_generate_upstream_server_config(spark_host_info_path):
7+
8+
format_tab = "\t"
9+
format_tab2 = format_tab*2
10+
up_stream_place_template = format_tab+"upstream server_${hostname} {"+os.linesep + \
11+
format_tab2 + "server ${host}:${port};" + os.linesep + \
12+
format_tab + "}"
13+
server_place_holder_template = format_tab+"server {" + os.linesep + \
14+
format_tab2 + "listen 80;" + os.linesep + \
15+
format_tab2 + "server_name ${hostname};"+os.linesep + \
16+
format_tab2 + "location / {" + os.linesep + \
17+
format_tab2 + " proxy_pass http://server_${hostname};" + os.linesep + \
18+
format_tab2 + "}"+os.linesep + \
19+
format_tab + "}"
20+
spark_host_info_file = open(spark_host_info_path)
21+
host_info_lines = spark_host_info_file.readlines()[1:]
22+
23+
up_stream_str = ""
24+
server_stream_str = ""
25+
26+
spark_master_host_name = "spark_master"
27+
up_stream_master_item = up_stream_place_template.replace("${hostname}", spark_master_host_name)\
28+
.replace("${host}", "127.0.0.1")\
29+
.replace("${port}", "8080").replace("\t", "", 1)
30+
server_stream_master_item = \
31+
server_place_holder_template.replace("${hostname}", spark_master_host_name).replace("\t", "", 1)
32+
33+
up_stream_str += up_stream_master_item + os.linesep
34+
server_stream_str += server_stream_master_item+os.linesep
35+
36+
for host_info in host_info_lines:
37+
host_info_list = host_info.split()
38+
up_stream_item = up_stream_place_template.replace("${hostname}", host_info_list[1].strip()) \
39+
.replace("${host}", host_info_list[0].strip()) \
40+
.replace("${port}", "8081")
41+
server_stream_item = server_place_holder_template.replace("${hostname}", host_info_list[1].strip())
42+
43+
up_stream_str += up_stream_item.rstrip() + os.linesep
44+
server_stream_str += server_stream_item.rstrip()+os.linesep
45+
return up_stream_str, server_stream_str
46+
47+
def do_update_nginx_config_file(result_content, nginx_config_target_path):
48+
nginx_config_file = file(nginx_config_target_path, "w")
49+
nginx_config_file.write(result_content)
50+
51+
def generate_config_file(spark_host_info_path,nginx_config_template_path, nginx_config_taget_path):
52+
53+
up_stream_place_holder="${upstream_place_holder}"
54+
server_place_holder="${server_place_holder}"
55+
56+
nginx_upstream_server_tuple = do_generate_upstream_server_config(spark_host_info_path)
57+
58+
nginx_config_template_file = open(nginx_config_template_path)
59+
nginx_config_template_lines = nginx_config_template_file.readlines()
60+
result_content = ""
61+
for line in nginx_config_template_lines:
62+
result_content += line
63+
64+
result_content = result_content.replace(up_stream_place_holder, nginx_upstream_server_tuple[0]) \
65+
.replace(server_place_holder, nginx_upstream_server_tuple[1])
66+
67+
do_update_nginx_config_file(result_content, nginx_config_taget_path)
68+

0 commit comments

Comments
 (0)