Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f559d64

Browse files
oubiwannoubiwann
committed
Updated and added modules.
1 parent b5d2190 commit f559d64

File tree

7 files changed

+254
-0
lines changed

7 files changed

+254
-0
lines changed

‎lib/all.py

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import multiprocessing, random, sys, time
2+
import zmq
3+
import democfg
4+
5+
def run_task(task_data):
6+
in_circle = 0
7+
for _ in range(int(task_data)):
8+
(x, y) = (random.random(), random.random())
9+
if (x ** 2 + y ** 2) <= 1:
10+
in_circle += 1
11+
return 4 * in_circle / task_data
12+
13+
def process_results(receiver):
14+
data = []
15+
# Process results from the workers
16+
for _ in range(democfg.task_count):
17+
result = receiver.recv_pyobj()
18+
print("Processing result: {}".format(result))
19+
data.append(result)
20+
# Calculate final result
21+
print("Final result: {}".format(sum(data)/len(data)))
22+
23+
def is_done(socks, controller):
24+
if (socks.get(controller) == zmq.POLLIN and
25+
controller.recv() == democfg.done_msg):
26+
return True
27+
return False
28+
29+
def work(worker_id):
30+
context = zmq.Context()
31+
# Socket to receive messages on
32+
receiver = context.socket(zmq.PULL)
33+
receiver.connect(democfg.routing_table["receiver"])
34+
# Socket to send messages to
35+
sender = context.socket(zmq.PUSH)
36+
sender.connect(democfg.routing_table["sender"])
37+
# Socket for control input
38+
controller = context.socket(zmq.SUB)
39+
controller.connect(democfg.routing_table["controller"])
40+
controller.setsockopt(zmq.SUBSCRIBE, b"")
41+
# Process messages from receiver and controller
42+
poller = zmq.Poller()
43+
poller.register(receiver, zmq.POLLIN)
44+
poller.register(controller, zmq.POLLIN)
45+
# Process messages from both sockets
46+
run_loop = True
47+
while run_loop:
48+
socks = dict(poller.poll())
49+
if socks.get(receiver) == zmq.POLLIN:
50+
task_data = receiver.recv_pyobj()
51+
# Process task data
52+
result = run_task(task_data)
53+
sender.send_pyobj(result)
54+
if is_done(socks, controller):
55+
run_loop = False
56+
57+
def sink():
58+
context = zmq.Context()
59+
# Socket to receive messages on (collect results from worker)
60+
receiver = context.socket(zmq.PULL)
61+
receiver.bind(democfg.routing_table["sender"])
62+
# Socket for worker control
63+
controller = context.socket(zmq.PUB)
64+
controller.bind(democfg.routing_table["controller"])
65+
# Wait for start signal
66+
assert receiver.recv() == b"0"
67+
process_results(receiver)
68+
# Let workers know that all results have been processed
69+
controller.send(democfg.done_msg)
70+
# Finished
71+
time.sleep(5 * democfg.pause_time) # Give 0MQ time to deliver
72+
73+
def vent(tasks):
74+
context = zmq.Context()
75+
# Socket to send messages on
76+
sender = context.socket(zmq.PUSH)
77+
sender.bind(democfg.routing_table["receiver"])
78+
# Socket with direct access to the sink used to syncronize start of batch
79+
syncher = context.socket(zmq.PUSH)
80+
syncher.connect(democfg.routing_table["sender"])
81+
# Socket for control input
82+
controller = context.socket(zmq.SUB)
83+
controller.connect(democfg.routing_table["controller"])
84+
controller.setsockopt(zmq.SUBSCRIBE, b"")
85+
# Give 0MQ time to start up
86+
time.sleep(democfg.pause_time)
87+
syncher.send(b"0")
88+
poller = zmq.Poller()
89+
poller.register(controller, zmq.POLLIN)
90+
for task in tasks:
91+
sender.send_pyobj(task)
92+
# Give 0MQ time to deliver message
93+
time.sleep(democfg.pause_time)
94+
95+
def main(tasks):
96+
# Initialize random number generator
97+
random.seed()
98+
print("Starting workers ...")
99+
for worker_id in range(democfg.worker_pool_size):
100+
worker = multiprocessing.Process(target=work, args=[worker_id])
101+
worker.start()
102+
print("Starting sink ...")
103+
sinker = multiprocessing.Process(target=sink)
104+
sinker.start()
105+
time.sleep(democfg.pause_time)
106+
print("Starting ventilator ...")
107+
vent(tasks)
108+
109+
if __name__ == "__main__":
110+
count = range(democfg.task_count)
111+
tasks = [1e8 / democfg.task_count for _ in count]
112+
main(tasks)

‎lib/collector.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import time
2+
import zmq
3+
import democfg
4+
5+
def process_results(receiver):
6+
data = []
7+
# Process results from the workers
8+
for _ in range(democfg.task_count):
9+
result = receiver.recv_pyobj()
10+
print("Processing result: {}".format(result))
11+
data.append(result)
12+
# Calculate final result
13+
print("Final result: {}".format(sum(data)/len(data)))
14+
15+
def collect():
16+
context = zmq.Context()
17+
# Socket to receive messages on (collect results from worker)
18+
receiver = context.socket(zmq.PULL)
19+
receiver.bind(democfg.routing_table["sender"])
20+
# Socket for worker control
21+
controller = context.socket(zmq.PUB)
22+
controller.bind(democfg.routing_table["controller"])
23+
# Wait for start signal
24+
assert receiver.recv() == democfg.start_flag
25+
process_results(receiver)
26+
# Let workers know that all results have been processed
27+
controller.send(democfg.done_msg)
28+
# Finished
29+
time.sleep(5 * democfg.pause_time) # Give 0MQ time to deliver

‎lib/demo.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import multiprocessing, random, time
2+
import zmq
3+
import collector, democfg, distributor, worker
4+
5+
def main(tasks):
6+
# Initialize random number generator
7+
random.seed()
8+
print("Starting task workers ...")
9+
for worker_id in range(democfg.worker_pool_size):
10+
worker_proc = multiprocessing.Process(
11+
target=worker.work, args=[worker_id])
12+
worker_proc.start()
13+
print("Starting task collector ...")
14+
collector_proc = multiprocessing.Process(target=collector.collect)
15+
collector_proc.start()
16+
time.sleep(democfg.pause_time)
17+
print("Starting task distributor ...")
18+
distributor.distribute(tasks)
19+
20+
if __name__ == "__main__":
21+
count = range(democfg.task_count)
22+
tasks = [1e8 / democfg.task_count for _ in count]
23+
main(tasks)

‎lib/democfg.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
worker_pool_size = 8
2+
task_count = 8
3+
delay_range = (1, 100) # milliseconds
4+
pause_time = 1 # seconds
5+
start_flag = b"0"
6+
done_msg = b"DONE"
7+
routing_table = {
8+
"receiver": "tcp://127.0.0.1:7557",
9+
"sender": "tcp://127.0.0.1:7558",
10+
"controller": "tcp://127.0.0.1:7559"}

‎lib/distributor.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import time
2+
import zmq
3+
import democfg
4+
5+
def distribute(tasks):
6+
context = zmq.Context()
7+
# Socket to send messages on
8+
sender = context.socket(zmq.PUSH)
9+
sender.bind(democfg.routing_table["receiver"])
10+
# Socket with direct access to the sink used to syncronize start of batch
11+
syncher = context.socket(zmq.PUSH)
12+
syncher.connect(democfg.routing_table["sender"])
13+
# Give 0MQ time to start up
14+
time.sleep(democfg.pause_time)
15+
syncher.send(democfg.start_flag)
16+
for task in tasks:
17+
sender.send_pyobj(task)
18+
# Give 0MQ time to deliver message
19+
time.sleep(democfg.pause_time)

‎lib/nbutil.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from pygments import highlight
2+
from pygments.lexers import PythonLexer
3+
from pygments.formatters import HtmlFormatter
4+
5+
import IPython
6+
7+
8+
def pycat(filename):
9+
with open(filename) as f:
10+
code = f.read()
11+
12+
formatter = HtmlFormatter()
13+
return IPython.display.HTML(
14+
'<style type="text/css">{}</style>{}'.format(
15+
formatter.get_style_defs(".highlight"),
16+
highlight(code, PythonLexer(), formatter)))

‎lib/worker.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import random
2+
import zmq
3+
import democfg
4+
5+
def run_task(task_data):
6+
in_circle = 0
7+
for _ in range(int(task_data)):
8+
(x, y) = (random.random(), random.random())
9+
if (x ** 2 + y ** 2) <= 1:
10+
in_circle += 1
11+
return 4 * in_circle / task_data
12+
13+
def is_done(socks, controller):
14+
if (socks.get(controller) == zmq.POLLIN and
15+
controller.recv() == democfg.done_msg):
16+
return True
17+
return False
18+
19+
def work(worker_id):
20+
context = zmq.Context()
21+
# Socket to receive messages on
22+
receiver = context.socket(zmq.PULL)
23+
receiver.connect(democfg.routing_table["receiver"])
24+
# Socket to send messages to
25+
sender = context.socket(zmq.PUSH)
26+
sender.connect(democfg.routing_table["sender"])
27+
# Socket for control input
28+
controller = context.socket(zmq.SUB)
29+
controller.connect(democfg.routing_table["controller"])
30+
controller.setsockopt(zmq.SUBSCRIBE, b"")
31+
# Process messages from receiver and controller
32+
poller = zmq.Poller()
33+
poller.register(receiver, zmq.POLLIN)
34+
poller.register(controller, zmq.POLLIN)
35+
# Process messages from both sockets
36+
run_loop = True
37+
while run_loop:
38+
socks = dict(poller.poll())
39+
if socks.get(receiver) == zmq.POLLIN:
40+
task_data = receiver.recv_pyobj()
41+
# Process task data
42+
result = run_task(task_data)
43+
sender.send_pyobj(result)
44+
if is_done(socks, controller):
45+
run_loop = False

0 commit comments

Comments
 (0)