Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5703bd0

Browse files
randerande
committed
Merge branch 'refactor_proxy'
2 parents e256c93 + ad61e51 commit 5703bd0

File tree

2 files changed

+149
-119
lines changed

2 files changed

+149
-119
lines changed

‎element/proxy.py

Lines changed: 132 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
1-
21
# References used to code this file
3-
# - https://github.com/senko/tornado-proxy/blob/master/tornado_proxy/proxy.py
42
# - https://raw.githubusercontent.com/tornadoweb/tornado/master/tornado/autoreload.py
53

6-
import socket
7-
import tornado.httpserver
8-
import tornado.web
9-
import tornado.httpclient
104
import os
115
import sys
126
import types
137
import fnmatch
14-
import re
8+
import logging
9+
import socket, re
10+
11+
import tornado.httpclient
12+
from tornado.iostream import IOStream
13+
from tornado.tcpserver import TCPServer
14+
from tornado.httputil import HTTPHeaders
15+
16+
gen_log = logging.getLogger("tornado.general")
1517

1618
try:
1719
from urllib.parse import urlparse
@@ -43,13 +45,13 @@ def start(self, io_loop=None, check_time=2000):
4345

4446
self.modify_times = {}
4547

46-
print("%d files to watch" % len(self._watched_files))
48+
gen_log.info("%d files to watch" % len(self._watched_files))
49+
4750
# tornado.ioloop.PeriodicCallback(self.build_watched_files, 5000, io_loop=self.io_loop).start()
4851
tornado.ioloop.PeriodicCallback(self._reload_on_update, check_time, io_loop=self.io_loop).start()
4952

5053
def build_watched_files(self):
51-
52-
print("Reloading watched files")
54+
gen_log.info("Reloading watched files")
5355
self._watched_files = set()
5456
for path in self.paths:
5557
for pattern in self.patterns:
@@ -120,140 +122,165 @@ def __init__(self, command):
120122
self.process = False
121123

122124
def start(self):
125+
gen_log.info("Start command: %s " % self.command)
123126
self.process = tornado.process.Subprocess(self.command, shell=False)
124127
self.process.set_exit_callback(self._restart)
125128

126129
def _restart(self, code):
127130
if self.reloading:
128-
print("Create a new process")
131+
gen_log.info("Create a new process !!")
129132
self.start()
130133
self.reloading = False
131134

132135
def restart(self, *args, **kwargs):
133136
self.reloading = True
134-
if self.process.proc.returncode != None:
135-
self.process.proc.terminate()
136137

137-
class ProxyHandler(tornado.web.RequestHandler):
138-
SUPPORTED_METHODS = ['GET', 'POST', 'CONNECT']
138+
if not self.process.proc.returncode:
139+
gen_log.info("Terminate process")
140+
self.process.proc.kill()
141+
142+
def parse_headers(data):
143+
headers = HTTPHeaders()
144+
145+
for line in data.splitlines():
146+
if line:
147+
try:
148+
headers.parse_line(line)
149+
except Exception, e:
150+
break
151+
152+
return headers
153+
154+
def is_websocket(headers):
155+
"""
156+
Detect if the data is related to a websocket connection, should be called only once
157+
158+
http://en.wikipedia.org/wiki/WebSocket
159+
"""
160+
return "Upgrade" in headers and headers["Upgrade"] == "websocket"
161+
162+
class StreamProxy(object):
163+
def __init__(self, io_source, io_target):
164+
self.public_io = io_source
165+
self.internal_io = io_target
166+
167+
self.reset()
168+
169+
def reset(self):
170+
self.init = False
171+
self.is_websocket = None
172+
self.is_html = None
173+
174+
self.public_headers = None
175+
self.internal_headers = None
176+
177+
self.content_length = 0
178+
179+
def handle(self):
180+
self.public_io.read_until_close(callback=self.end_public, streaming_callback=self.proxy_data_to_internal)
181+
self.internal_io.read_until_close(callback=self.end_internal, streaming_callback=self.proxy_data_to_public)
182+
183+
def end_public(self, data):
184+
gen_log.debug("Proxy > callback end_public")
185+
self.internal_io.close()
186+
187+
def end_internal(self, data):
188+
gen_log.debug("Proxy > callback end_internal")
189+
self.public_io.close()
190+
191+
def proxy_data_to_internal(self, data):
192+
"""
193+
This method is used to send streamed data to the internal webserver
194+
"""
195+
if not self.init:
196+
self.init = True
197+
198+
request, headers = data.split("\r\n", 1)
199+
gen_log.info("request: %s" % request)
139200

140-
def initialize(self, state, proxy=None):
141-
self.state = state
142-
self.client = tornado.httpclient.AsyncHTTPClient()
143-
self.proxy = proxy or 'localhost:5001'
201+
self.public_headers = parse_headers(headers)
202+
self.is_websocket = is_websocket(self.public_headers)
144203

145-
def handle_response(self, response):
146-
if response.error and not isinstance(response.error, tornado.httpclient.HTTPError):
147-
self.set_status(500)
148-
self.write('Internal server error:\n' + str(response.error))
149-
else:
150-
self.set_status(response.code)
204+
if not self.is_websocket:
205+
# we don't want to deal with gzip content
206+
data = data.replace("Accept-Encoding: gzip, deflate\r\n", "")
151207

152-
for header in ('Date', 'Cache-Control', 'Server', 'Content-Type', 'Location'):
153-
v = response.headers.get(header)
208+
self.internal_io.write(data)
154209

155-
if v:
156-
self.set_header(header, v)
210+
def close_stream(self):
211+
if self.internal_headers and 'Content-Length' in self.internal_headers and self.content_length >= int(self.internal_headers['Content-Length']):
212+
if self.public_io.writing():
213+
gen_log.debug(" > still data being written")
157214

158-
if not response.body:
159215
return
160216

161-
if response.headers['Content-Type'].startswith('text/html'):
162-
self.write(self.replace_tag(response))
163-
else:
164-
self.write(response.body)
217+
gen_log.debug(" > closing all streams")
165218

166-
self.finish()
219+
self.public_io.close()
220+
self.internal_io.close()
167221

168-
def replace_tag(self, response):
222+
self.reset()
223+
224+
def proxy_data_to_public(self, data):
225+
if self.is_websocket:
226+
self.public_io.write(data)
227+
return
228+
229+
if not self.internal_headers:
230+
# we parse the response to replace esi tag
231+
self.internal_headers = parse_headers(data.split("\r\n", 1)[1])
232+
233+
self.is_html = 'Content-Type' in self.internal_headers and self.internal_headers['Content-Type'].startswith('text/html')
234+
235+
self.content_length += len(data)
236+
237+
if self.is_html and 'Content-Length' in self.internal_headers:
238+
data = data.replace('Content-Length:', 'X-Content-Length:')
239+
data = self.replace_tag(data)
240+
241+
self.public_io.write(data, callback=self.close_stream)
242+
243+
def replace_tag(self, data):
169244
client = tornado.httpclient.HTTPClient()
170-
headers = self.request.headers.copy()
245+
headers = self.public_headers.copy()
171246
headers['Surrogate-Capability'] = 'abc=ESI/1.0'
172247

173-
if 'Set-Cookie' in response.headers:
174-
headers['Cookie'] = response.headers['Set-Cookie']
248+
if 'Set-Cookie' in self.internal_headers:
249+
headers['Cookie'] = self.internal_headers['Set-Cookie']
250+
251+
if 'Accept-Encoding' in headers:
252+
# we don't want to deal with gzip content
253+
del(headers['Accept-Encoding'])
175254

176255
def get_contents(matcher):
177256
result = urlparse(matcher.group(2))
178257

179-
url = "http://%s%s?%s" % (self.proxy, result.path, result.query)
258+
url = "http://%s%s?%s" % ("localhost:5001", result.path, result.query)
180259

181260
try:
182-
print("FETCHING ESI TAG: %s" % url)
261+
gen_log.info(" > start sub-request: %s" % url)
183262

184263
sub_response = client.fetch(tornado.httpclient.HTTPRequest(url, headers=headers))
185264

265+
gen_log.debug(" > end sub-request: %s" % url)
266+
186267
return sub_response.body
187268
except tornado.httpclient.HTTPError as e:
188269
return "<!-- error resolving : %s !-->" % url
189270

190-
content = re.sub(r"<esi:include\W[^>]*src=(\"|')([^\"']*)(\"|')[^>]*/>", get_contents, response.body, flags=re.IGNORECASE)
271+
content = re.sub(r"<esi:include\W[^>]*src=(\"|')([^\"']*)(\"|')[^>]*/>", get_contents, data, flags=re.IGNORECASE)
191272

192273
return content
193274

194-
@tornado.web.asynchronous
195-
def get(self):
196-
197-
if self.state.reloading:
198-
self.write("reloading")
275+
class ProxyTCPServer(TCPServer):
276+
def __init__(self, sub_child_port, **kwargs):
277+
self.sub_child_port = sub_child_port
199278

200-
self.finish()
201-
return
202-
203-
url = "%s://%s%s" % ('http', self.proxy, self.request.uri)
204-
205-
print("PROXY: Task: %s - %s %s" % (tornado.process.task_id(), self.request.method, url))
206-
207-
req = tornado.httpclient.HTTPRequest(url=url,
208-
method=self.request.method, body=self.request.body,
209-
headers=self.request.headers, follow_redirects=False,
210-
allow_nonstandard_methods=True
211-
)
212-
213-
try:
214-
self.client.fetch(req, self.handle_response)
215-
except tornado.httpclient.HTTPError as e:
216-
if hasattr(e, 'response') and e.response:
217-
self.handle_response(e.response)
218-
else:
219-
self.set_status(500)
220-
self.write('Internal server error:\n' + str(e))
221-
self.finish()
222-
223-
@tornado.web.asynchronous
224-
def post(self):
225-
return self.get()
226-
227-
@tornado.web.asynchronous
228-
def connect(self):
229-
host, port = self.request.uri.split(':')
230-
client = self.request.connection.stream
231-
232-
def read_from_client(data):
233-
upstream.write(data)
234-
235-
def read_from_upstream(data):
236-
client.write(data)
237-
238-
def client_close(data=None):
239-
if upstream.closed():
240-
return
241-
if data:
242-
upstream.write(data)
243-
upstream.close()
244-
245-
def upstream_close(data=None):
246-
if client.closed():
247-
return
248-
if data:
249-
client.write(data)
250-
client.close()
251-
252-
def start_tunnel():
253-
client.read_until_close(client_close, read_from_client)
254-
upstream.read_until_close(upstream_close, read_from_upstream)
255-
client.write(b'HTTP/1.0 200 Connection established\r\n\r\n')
279+
super(ProxyTCPServer, self).__init__(**kwargs)
256280

281+
def handle_stream(self, io_source, address):
257282
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
258-
upstream = tornado.iostream.IOStream(s)
259-
upstream.connect((host, int(port)), start_tunnel)
283+
io_target = IOStream(s)
284+
285+
proxy = StreamProxy(io_source, io_target)
286+
io_target.connect(("localhost", self.sub_child_port), proxy.handle)

‎element/standalone/skeleton/proxy.py

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,15 @@
44
# rendering esi:include tag.
55

66
import sys, os, tornado
7-
from element.proxy import ProxyHandler, ProxyState, FilesWatcher
7+
from element.proxy import ProxyState, FilesWatcher, ProxyTCPServer
8+
import logging
89

9-
if __name__ == '__main__':
10+
logging.basicConfig(format="[%(asctime)-15s] proxy.%(levelname)s: %(message)s")
11+
12+
gen_log = logging.getLogger("tornado.general")
13+
gen_log.setLevel(logging.INFO)
1014

15+
if __name__ == '__main__':
1116
port = 5000
1217
if len(sys.argv) > 1:
1318
port = int(sys.argv[1])
@@ -18,13 +23,15 @@
1823

1924
child_process = [
2025
'%s' % (sys.executable), 'start.py', 'tornado:start',
21-
# debug parameters
26+
# Debug parameters
2227
'--verbose', '-d',
23-
# start only one child
28+
# Start only one child, otherwise the Suprocess module will not be able to
29+
# properly kill sub children process
30+
# There is no need to have more than once ...
2431
'-np', '1',
25-
# the subprocess will listen to port 5001, and the master to 5001
32+
# The subprocess will listen to port 5001, and the master to 5001
2633
'-p', str(sub_child_port),
27-
# the bind parameter is used to define the host used to render absolute urls
34+
# The bind parameter is used to define the host used to render absolute urls
2835
'--bind', 'element.vagrant:%d' % port
2936
]
3037

@@ -33,18 +40,14 @@
3340

3441
state = ProxyState(child_process)
3542

36-
app = tornado.web.Application([
37-
(r'.*', ProxyHandler, dict(state=state, proxy='localhost:%d' % sub_child_port)),
38-
])
39-
40-
server = tornado.httpserver.HTTPServer(app)
41-
server.bind(port, '0.0.0.0')
42-
server.start(8)
43+
server = ProxyTCPServer(sub_child_port)
44+
server.bind(port)
45+
server.start(1) # Forks multiple sub-processes
4346

4447
if tornado.process.task_id() == 0 or not tornado.process.task_id():
4548
w = FilesWatcher([
4649
os.path.dirname(os.path.abspath(__file__)),
47-
'/home/vagrant/python/element'
50+
# '/home/vagrant/python/element'
4851
# add more path to watch
4952
])
5053

0 commit comments

Comments
 (0)