Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 6e2e50c

Browse files
binuxbinux
committed
join every thread, change phantomjs webui port to 25001
1 parent d345f20 commit 6e2e50c

11 files changed

+53
-32
lines changed

‎tests/test_bench.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@
66
# Created on 2014-12-10 01:34:09
77

88
import os
9-
import sys
10-
import time
11-
import click
9+
import signal
1210
import shutil
1311
import inspect
1412
import unittest2 as unittest
@@ -36,7 +34,7 @@ def test_10_bench(self):
3634
'--queue-maxsize=0',
3735
'bench',
3836
'--total=500'
39-
], close_fds=True, stderr=subprocess.PIPE)
37+
], close_fds=True, stderr=subprocess.PIPE, preexec_fn=os.setsid)
4038

4139
stdout, stderr = p.communicate()
4240
stderr = utils.text(stderr)
@@ -47,3 +45,8 @@ def test_10_bench(self):
4745
self.assertIn('Fetched', stderr)
4846
self.assertIn('Processed', stderr)
4947
self.assertIn('Saved', stderr)
48+
49+
try:
50+
os.killpg(p.pid, signal.SIGTERM)
51+
except OSError:
52+
pass

‎tests/test_fetcher.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import json
1010
import copy
1111
import time
12+
import signal
1213
import umsgpack
1314
import subprocess
1415
import unittest2 as unittest
@@ -73,7 +74,7 @@ def setUpClass(self):
7374
self.phantomjs = subprocess.Popen(['phantomjs',
7475
os.path.join(os.path.dirname(__file__),
7576
'../pyspider/fetcher/phantomjs_fetcher.js'),
76-
'25555'])
77+
'25555'], preexec_fn=os.setsid)
7778
except OSError:
7879
self.phantomjs = None
7980
time.sleep(0.5)
@@ -83,13 +84,15 @@ def tearDownClass(self):
8384
self.proxy_thread.terminate()
8485
self.proxy_thread.wait()
8586
self.httpbin_thread.terminate()
86-
self.httpbin_thread.join()
87+
self.httpbin_thread.join(5)
88+
self.xmlrpc_thread.join(5)
8789

8890
if self.phantomjs:
8991
self.phantomjs.kill()
9092
self.phantomjs.wait()
9193
self.rpc._quit()
92-
self.thread.join()
94+
self.xmlrpc_thread.join(5)
95+
self.thread.join(5)
9396
time.sleep(1)
9497

9598
def test_10_http_get(self):

‎tests/test_fetcher_processor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ def tearDownClass(self):
5151
self.proxy_thread.terminate()
5252
self.proxy_thread.wait()
5353
self.httpbin_thread.terminate()
54-
self.httpbin_thread.join()
54+
self.httpbin_thread.join(5)
5555

5656
def crawl(self, url=None, track=None, **kwargs):
5757
if url is None and kwargs.get('callback'):

‎tests/test_processor.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,8 +222,7 @@ def run_processor():
222222
def tearDownClass(self):
223223
if self.process.is_alive():
224224
self.processor.quit()
225-
self.process.join(2)
226-
assert not self.process.is_alive()
225+
self.process.join(5)
227226
shutil.rmtree('./data/tests/', ignore_errors=True)
228227

229228
def test_10_update_project(self):

‎tests/test_response.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ def setUpClass(self):
3737
@classmethod
3838
def tearDownClass(self):
3939
self.httpbin_thread.terminate()
40+
self.httpbin_thread.join(5)
4041

4142
def get(self, url, **kwargs):
4243
if not url.startswith('http://'):

‎tests/test_result_worker.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ def run_result_worker():
4141
def tearDownClass(self):
4242
if self.process.is_alive():
4343
self.result_worker.quit()
44-
self.process.join(2)
44+
self.process.join(5)
4545
assert not self.process.is_alive()
4646
shutil.rmtree('./data/tests/', ignore_errors=True)
4747

‎tests/test_run.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def setUpClass(self):
3737
@classmethod
3838
def tearDownClass(self):
3939
self.httpbin_thread.terminate()
40-
self.httpbin_thread.join()
40+
self.httpbin_thread.join(5)
4141

4242
shutil.rmtree('./data/tests', ignore_errors=True)
4343

@@ -195,7 +195,6 @@ def test_a100_all(self):
195195
'all',
196196
], close_fds=True, preexec_fn=os.setsid)
197197

198-
199198
try:
200199
limit = 30
201200
while limit >= 0:
@@ -310,16 +309,17 @@ def setUpClass(self):
310309

311310
ctx = run.scheduler.make_context('scheduler', [], self.ctx)
312311
scheduler = run.scheduler.invoke(ctx)
313-
utils.run_in_thread(scheduler.xmlrpc_run)
314-
utils.run_in_thread(scheduler.run)
312+
self.scheduler_xmlrpc_p = utils.run_in_thread(scheduler.xmlrpc_run)
313+
self.scheduler_p = utils.run_in_thread(scheduler.run)
315314

316315
time.sleep(1)
317316

318317
@classmethod
319318
def tearDownClass(self):
320319
for each in self.ctx.obj.instances:
321320
each.quit()
322-
time.sleep(1)
321+
self.scheduler_xmlrpc_p.join(5)
322+
self.scheduler_p.join(5)
323323

324324
shutil.rmtree('./data/tests', ignore_errors=True)
325325

‎tests/test_scheduler.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ def run_scheduler():
140140
scheduler.INQUEUE_LIMIT = 10
141141
Scheduler.DELETE_TIME = 0
142142
scheduler._last_tick = int(time.time()) # not dispatch cronjob
143-
run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port)
143+
self.scheduler_xmlrpc_p = run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port)
144144
scheduler.run()
145145

146146
self.process = run_in_thread(run_scheduler)
@@ -151,6 +151,7 @@ def tearDownClass(self):
151151
if self.process.is_alive():
152152
self.rpc._quit()
153153
self.process.join(5)
154+
self.scheduler_xmlrpc_p.join(5)
154155
assert not self.process.is_alive()
155156
shutil.rmtree('./data/tests', ignore_errors=True)
156157
time.sleep(1)

‎tests/test_webdav.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ def setUpClass(self):
3838
'--password', '4321',
3939
], self.ctx)
4040
self.app = run.webui.invoke(ctx)
41-
utils.run_in_thread(self.app.run)
41+
self.p = utils.run_in_thread(self.app.run)
4242
time.sleep(5)
4343

4444
self.webdav = easywebdav.connect('localhost', port=5000, path='dav')
@@ -49,6 +49,7 @@ def setUpClass(self):
4949
def tearDownClass(self):
5050
for each in self.ctx.obj.instances:
5151
each.quit()
52+
self.p.join(5)
5253
time.sleep(1)
5354

5455
shutil.rmtree('./data/tests', ignore_errors=True)

‎tests/test_webui.js

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@
33
*/
44

55

6+
var base_url = casper.cli.options.base || "http://localhost:5000/";
7+
68
casper.test.begin('index page', function suite(test) {
79
test.info('open index.html');
8-
casper.start('http://localhost:5000/', function() {
10+
casper.start(base_url, function() {
911
test.assertSelectorHasText('header > h1', 'pyspider dashboard');
1012
test.assertExists('.project-create');
1113

@@ -32,7 +34,7 @@ casper.test.begin('index page', function suite(test) {
3234
}, 1000);
3335

3436
casper.then(function() {
35-
test.assertUrlMatch(/^http:\/\/localhost:5000\/debug\/test_project/);
37+
test.assert(this.getCurrentUrl().indexOf(base_url+'debug/test_project') == 0);
3638
});
3739

3840
casper.run(function() {
@@ -42,7 +44,7 @@ casper.test.begin('index page', function suite(test) {
4244

4345
casper.test.begin('debug page', function(test) {
4446
test.info('open index.html');
45-
casper.start('http://localhost:5000/', function() {
47+
casper.start(base_url, function() {
4648
test.info('fill and create project');
4749
this.fill('#create-project-modal form', {
4850
'project-name': 'test_project',

‎tests/test_webui.py

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,20 +38,20 @@ def setUpClass(self):
3838

3939
ctx = run.scheduler.make_context('scheduler', [], self.ctx)
4040
scheduler = run.scheduler.invoke(ctx)
41-
run_in_thread(scheduler.xmlrpc_run)
42-
run_in_thread(scheduler.run)
41+
self.scheduler_xmlrpc_p = run_in_thread(scheduler.xmlrpc_run)
42+
self.scheduler_p = run_in_thread(scheduler.run)
4343

4444
ctx = run.fetcher.make_context('fetcher', [], self.ctx)
4545
fetcher = run.fetcher.invoke(ctx)
46-
run_in_thread(fetcher.run)
46+
self.fetcher_p = run_in_thread(fetcher.run)
4747

4848
ctx = run.processor.make_context('processor', [], self.ctx)
4949
processor = run.processor.invoke(ctx)
50-
run_in_thread(processor.run)
50+
self.processor_p = run_in_thread(processor.run)
5151

5252
ctx = run.result_worker.make_context('result_worker', [], self.ctx)
5353
result_worker = run.result_worker.invoke(ctx)
54-
run_in_thread(result_worker.run)
54+
self.result_worker_p = run_in_thread(result_worker.run)
5555

5656
ctx = run.webui.make_context('webui', [
5757
'--scheduler-rpc', 'http://localhost:23333/'
@@ -67,10 +67,14 @@ def setUpClass(self):
6767
def tearDownClass(self):
6868
for each in self.ctx.obj.instances:
6969
each.quit()
70-
time.sleep(1)
70+
self.scheduler_xmlrpc_p.join(5)
71+
self.scheduler_p.join(5)
72+
self.fetcher_p.join(5)
73+
self.processor_p.join(5)
74+
self.result_worker_p.join(5)
7175

7276
self.httpbin_thread.terminate()
73-
self.httpbin_thread.join()
77+
self.httpbin_thread.join(3)
7478

7579
shutil.rmtree('./data/tests', ignore_errors=True)
7680

@@ -474,12 +478,18 @@ def setUpClass(self):
474478
time.sleep(3)
475479
shutil.rmtree('./data/tests', ignore_errors=True)
476480
os.makedirs('./data/tests')
481+
json.dump({
482+
'taskdb': 'sqlite+taskdb:///data/tests/task.db',
483+
'projectdb': 'sqlite+projectdb:///data/tests/projectdb.db',
484+
'resultdb': 'sqlite+resultdb:///data/tests/resultdb.db',
485+
'webui': {
486+
'port': 25001,
487+
}
488+
}, fp=open('./data/tests/config.json', 'w'))
477489
cmd = [sys.executable]
478490
self.p = subprocess.Popen(cmd+[
479491
inspect.getsourcefile(run),
480-
'--taskdb', 'sqlite+taskdb:///data/tests/task.db',
481-
'--projectdb', 'sqlite+projectdb:///data/tests/projectdb.db',
482-
'--resultdb', 'sqlite+resultdb:///data/tests/resultdb.db',
492+
'-c', './data/tests/config.json',
483493
'all', ], close_fds=True, preexec_fn=os.setsid)
484494
time.sleep(3)
485495

@@ -491,4 +501,5 @@ def tearDownClass(self):
491501

492502
def test_10_webui(self):
493503
webui_js = os.path.join(os.path.dirname(__file__), 'test_webui.js')
494-
self.assertEqual(subprocess.check_call(['casperjs', 'test', webui_js]), 0)
504+
self.assertEqual(subprocess.check_call(['casperjs', 'test', webui_js,
505+
'--base=http://localhost:25001/']), 0)

0 commit comments

Comments
 (0)