Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 3d29026

Browse files
PeterDingPeterDing
committed
[tumblr] change threading to multiprocessing
1 parent 978ad77 commit 3d29026

1 file changed

Lines changed: 36 additions & 63 deletions

File tree

‎tumblr.py‎

Lines changed: 36 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@
88
import re
99
import json
1010
import collections
11-
import Queue
12-
import threading
11+
import multiprocessing
1312
import requests
1413
import argparse
1514
import random
@@ -22,12 +21,12 @@
2221
PID_PATH = '/tmp/tumblr.py.pid'
2322

2423
# statistic parameters
25-
NET_ERRORS = 0
26-
UNCOMPLETION = 0
27-
DOWNLOAD_ERRORS = 0
28-
DOWNLOADS = 0
29-
COMPLETION = 0
30-
OFFSET = 0
24+
NET_ERRORS = multiprocessing.Value('i', 0)
25+
UNCOMPLETION = multiprocessing.Value('i', 0)
26+
DOWNLOAD_ERRORS = multiprocessing.Value('i', 0)
27+
DOWNLOADS = multiprocessing.Value('i', 0)
28+
COMPLETION = multiprocessing.Value('i', 0)
29+
OFFSET = multiprocessing.Value('i', 0)
3130

3231
############################################################
3332
# wget exit status
@@ -69,19 +68,12 @@ def __str__(self):
6968
return self.msg
7069

7170
def reset_statistic_params():
72-
global NET_ERRORS
73-
global UNCOMPLETION
74-
global DOWNLOAD_ERRORS
75-
global DOWNLOADS
76-
global COMPLETION
77-
global OFFSET
78-
79-
NET_ERRORS = 0
80-
UNCOMPLETION = 0
81-
DOWNLOAD_ERRORS = 0
82-
DOWNLOADS = 0
83-
COMPLETION = 0
84-
OFFSET = 0
71+
NET_ERRORS.value = 0
72+
UNCOMPLETION.value = 0
73+
DOWNLOAD_ERRORS.value = 0
74+
DOWNLOADS.value = 0
75+
COMPLETION.value = 0
76+
OFFSET.value = 0
8577

8678
def play(urls, args):
8779
for url in urls:
@@ -138,20 +130,16 @@ def download_run(item):
138130
def callback(filepath):
139131
os.rename('%s.tmp' % filepath, filepath)
140132

141-
class Downloader(threading.Thread):
133+
class Downloader(multiprocessing.Process):
142134
def __init__(self, queue, lock):
143-
threading.Thread.__init__(self)
135+
super(Downloader, self).__init__()
144136
self.queue = queue
145137
self.daemon = True
146138
self.lock = lock
147139

148140
def run(self):
149-
global UNCOMPLETION
150-
global DOWNLOADS
151-
global DOWNLOAD_ERRORS
152141
while True:
153142
item = self.queue.get()
154-
self.queue.task_done()
155143
if not item:
156144
break
157145
status = download_run(item)
@@ -161,12 +149,12 @@ def run(self):
161149
if status != 0:
162150
# print s % (1, 93, '[Error %s] at wget' % status), wget_es[status]
163151
self.lock.acquire()
164-
UNCOMPLETION += 1
165-
DOWNLOAD_ERRORS += 1
152+
UNCOMPLETION.value += 1
153+
DOWNLOAD_ERRORS.value += 1
166154
self.lock.release()
167155
else:
168156
self.lock.acquire()
169-
DOWNLOADS += 1
157+
DOWNLOADS.value += 1
170158
self.lock.release()
171159
callback(filepath)
172160

@@ -175,7 +163,6 @@ def _request(self, base_hostname, target, type, params):
175163
api_url = '/'.join(['https://api.tumblr.com/v2/blog',
176164
base_hostname, target, type])
177165
params['api_key'] = API_KEY
178-
global NET_ERRORS
179166
while True:
180167
try:
181168
res = ss.get(api_url, params=params, timeout=10)
@@ -184,7 +171,7 @@ def _request(self, base_hostname, target, type, params):
184171
except KeyboardInterrupt:
185172
sys.exit()
186173
except Exception as e:
187-
NET_ERRORS += 1 # count errors
174+
NET_ERRORS.value += 1 # count errors
188175
# print s % (1, 93, '[Error at requests]:'), e
189176
time.sleep(5)
190177
if json_data['meta']['msg'].lower() != 'ok':
@@ -336,13 +323,11 @@ def init_infos(self, base_hostname, target_type, tag=''):
336323
os.makedirs(subdir)
337324

338325
if not self.args.play:
339-
global UNCOMPLETION
340-
global COMPLETION
341326
for fl in os.listdir(subdir):
342327
if not fl.endswith('.tmp'):
343-
COMPLETION += 1
328+
COMPLETION.value += 1
344329
else:
345-
UNCOMPLETION += 1
330+
UNCOMPLETION.value += 1
346331

347332
if self.args.offset:
348333
self.offset = self.args.offset
@@ -488,8 +473,7 @@ def parse_urls(self, url):
488473
return self.download_photos(base_hostname, post_id=post_id, tag=self.args.tag)
489474

490475
def get_item_generator(self):
491-
global OFFSET
492-
OFFSET = self.offset
476+
OFFSET.value = self.offset
493477
items = self.make_items()
494478
for item in items:
495479
item['dir_'] = self.infos['dir_']
@@ -520,42 +504,34 @@ def args_handler(argv):
520504
help='update new things')
521505
p.add_argument('--redownload', action='store_true',
522506
help='redownload all things')
523-
#global args
524507
args = p.parse_args(argv[1:])
525508
xxx = args.xxx
526509

527510
if args.redownload: args.update = True
528511
return args, xxx
529512

530513
def print_msg(check):
531-
global NET_ERRORS
532-
global UNCOMPLETION
533-
global COMPLETION
534-
global DOWNLOADS
535-
global DOWNLOAD_ERRORS
536-
global OFFSET
537-
538514
time.sleep(2) # initial interval
539515

540516
while True:
541517
msg = "\r%s, %s, %s, %s, %s " % \
542518
(
543-
'D: ' + s % (1, 92, DOWNLOADS),
544-
'R: ' + s % (1, 93, UNCOMPLETION \
519+
'D: ' + s % (1, 92, DOWNLOADS.value),
520+
'R: ' + s % (1, 93, UNCOMPLETION.value \
545521
if not check \
546-
else UNCOMPLETION - DOWNLOAD_ERRORS - DOWNLOADS),
547-
'C: ' + s % (1, 97, COMPLETION + DOWNLOADS),
548-
'NE: ' + s % (1, 91, NET_ERRORS),
549-
'O: %s' % OFFSET
522+
else UNCOMPLETION.value - DOWNLOAD_ERRORS.value - DOWNLOADS.value),
523+
'C: ' + s % (1, 97, COMPLETION.value + DOWNLOADS.value),
524+
'NE: ' + s % (1, 91, NET_ERRORS.value),
525+
'O: %s' % OFFSET.value
550526
)
551527
sys.stdout.write(msg)
552528
sys.stdout.flush()
553529
time.sleep(2)
554530

555531
def sighandler(signum, frame):
556-
print s % (1, 91, "\n !! Signal:"), signum
557-
#print s % (1, 91, " !! Frame: %s" % frame)
558-
sys.exit(1)
532+
# print s % (1, 91, "\n !! Signal:"), signum
533+
# print s % (1, 91, " !! Frame: %s" % frame)
534+
sys.exit()
559535

560536
def handle_signal():
561537
signal.signal(signal.SIGBUS, sighandler)
@@ -573,26 +549,23 @@ def handle_signal():
573549
signal.signal(signal.SIGTERM, sighandler)
574550

575551
def main(argv):
576-
# Only main thread can capture signal,
577-
# all child threads not be affected
578-
# whether they are daemonic or not.
579552
handle_signal()
580553
args, xxx = args_handler(argv)
581554

582555
if args.play:
583556
play(xxx, args)
584557

585-
lock = threading.Lock()
586-
queue = Queue.Queue(maxsize=args.processes)
558+
lock = multiprocessing.Lock()
559+
queue = multiprocessing.Queue(maxsize=args.processes)
587560
thrs = []
588561
for i in range(args.processes):
589562
thr = Downloader(queue, lock)
590563
thr.start()
591564
thrs.append(thr)
592565

593566
# massage thread
594-
msg_thr = threading.Thread(target=print_msg, args=(args.check,))
595-
msg_thr.setDaemon(True)
567+
msg_thr = multiprocessing.Process(target=print_msg, args=(args.check,))
568+
msg_thr.daemon = True
596569
msg_thr.start()
597570

598571
for url in xxx:
@@ -633,7 +606,7 @@ def main(argv):
633606
for thr in thrs:
634607
thr.join()
635608

636-
msg_thr._Thread__stop()
609+
msg_thr.terminate()
637610

638611
if __name__ == '__main__':
639612
argv = sys.argv

0 commit comments

Comments
 (0)