From 8a0dc2327cc132c6b8142f4f45a6d646312a97a6 Mon Sep 17 00:00:00 2001 From: Alexander Else Date: Wed, 20 Feb 2013 10:23:48 +1100 Subject: [PATCH 001/113] Update README.rst minor spelling and continuity corrections --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 481a513..cdcf386 100644 --- a/README.rst +++ b/README.rst @@ -188,9 +188,9 @@ Now, in another terminal, let's try connecting to our awesome zeroservice:: $ zerorpc -j tcp://:4242 add_42 1 43 - $ zerorpc tcp://:4242 add_man 'I own a mint-condition Wolkswagen Golf' - "I own a mint-condition Wolkswagen Gold, man!" - $ zerorpc tcp://:4242 boat 'I own a mint-condition Wolkswagen Gold, man!' + $ zerorpc tcp://:4242 add_man 'I own a mint-condition Volkswagen Golf' + "I own a mint-condition Volkswagen Golf, man!" + $ zerorpc tcp://:4242 boat 'I own a mint-condition Volkswagen Golf, man!' "I'm on a boat!" From cf3741c3ce70d862c3b5ec2d8ef57e09c421228e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Wed, 6 Mar 2013 20:49:47 -0800 Subject: [PATCH 002/113] Follow updates on pyzmq 13.0.0 - XREP/XREQ constants are now definitively replaced by ROUTER/DEALER. - Some weird getattr/setattr magic were added to pyzmq, which makes inheriting from pyzmq classes annoyingly frustrating, especially: - zmq.Context: solved by defined attributes wiht None, at the class level. - zmq.Socket: solved by using self.__dict__ directly. --- tests/test_buffered_channel.py | 36 +++++++++++++++++----------------- tests/test_channel.py | 12 ++++++------ tests/test_events.py | 4 ++-- tests/test_heartbeat.py | 32 +++++++++++++++--------------- tests/test_server.py | 4 ++-- tests/test_wrapped_events.py | 12 ++++++------ tests/zmqbug.py | 2 +- zerorpc/context.py | 10 ++++++++++ zerorpc/core.py | 4 ++-- zerorpc/events.py | 15 ++++++++------ zerorpc/gevent_zmq.py | 16 +++++++++------ 11 files changed, 82 insertions(+), 65 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index b223640..ec67f2c 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -33,11 +33,11 @@ def test_close_server_bufchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -65,11 +65,11 @@ def test_close_server_bufchan(): def test_close_client_bufchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -97,11 +97,11 @@ def test_close_client_bufchan(): def test_heartbeat_can_open_channel_server_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -127,11 +127,11 @@ def test_heartbeat_can_open_channel_server_close(): def test_heartbeat_can_open_channel_client_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -157,11 +157,11 @@ def test_heartbeat_can_open_channel_client_close(): def test_do_some_req_rep(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -201,11 +201,11 @@ def server_do(): def test_do_some_req_rep_lost_server(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -247,11 +247,11 @@ def server_do(): def test_do_some_req_rep_lost_client(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -294,11 +294,11 @@ def server_do(): def test_do_some_req_rep_client_timeout(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -345,11 +345,11 @@ class CongestionError(Exception): def test_congestion_control_server_pushing(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) diff --git a/tests/test_channel.py b/tests/test_channel.py index 4eebd65..d58d862 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -30,11 +30,11 @@ def test_events_channel_client_side(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -55,11 +55,11 @@ def test_events_channel_client_side(): def test_events_channel_client_side_server_send_many(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -82,11 +82,11 @@ def test_events_channel_client_side_server_send_many(): def test_events_channel_both_side(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) diff --git a/tests/test_events.py b/tests/test_events.py index 3d4766a..537adfc 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -131,10 +131,10 @@ def test_events_req_rep2(): def test_events_dealer_router(): endpoint = random_ipc_endpoint() - server = zerorpc.Events(zmq.XREP) + server = zerorpc.Events(zmq.ROUTER) server.bind(endpoint) - client = zerorpc.Events(zmq.XREQ) + client = zerorpc.Events(zmq.DEALER) client.connect(endpoint) for i in xrange(6): diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 13caab5..9ba1c85 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -33,11 +33,11 @@ def test_close_server_hbchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -63,11 +63,11 @@ def test_close_server_hbchan(): def test_close_client_hbchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -93,11 +93,11 @@ def test_close_client_hbchan(): def test_heartbeat_can_open_channel_server_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -121,11 +121,11 @@ def test_heartbeat_can_open_channel_server_close(): def test_heartbeat_can_open_channel_client_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -149,11 +149,11 @@ def test_heartbeat_can_open_channel_client_close(): def test_do_some_req_rep(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -191,11 +191,11 @@ def server_do(): def test_do_some_req_rep_lost_server(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -235,11 +235,11 @@ def server_do(): def test_do_some_req_rep_lost_client(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -280,11 +280,11 @@ def server_do(): def test_do_some_req_rep_client_timeout(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) diff --git a/tests/test_server.py b/tests/test_server.py index 3f74679..e6e2ddf 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -46,7 +46,7 @@ def add(self, a, b): srv.bind(endpoint) gevent.spawn(srv.run) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -176,7 +176,7 @@ class MySrv(zerorpc.Server): srv.bind(endpoint) gevent.spawn(srv.run) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py index c631f6d..64024f2 100644 --- a/tests/test_wrapped_events.py +++ b/tests/test_wrapped_events.py @@ -32,11 +32,11 @@ def test_sub_events(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -64,11 +64,11 @@ def test_sub_events(): def test_multiple_sub_events(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) @@ -118,11 +118,11 @@ def test_multiple_sub_events(): def test_recursive_multiplexer(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) servermux = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) clientmux = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) diff --git a/tests/zmqbug.py b/tests/zmqbug.py index 2d385f0..83b335b 100644 --- a/tests/zmqbug.py +++ b/tests/zmqbug.py @@ -130,7 +130,7 @@ def responder(): def client(): - socket = ZMQSocket(zmq_context, zmq.XREQ) + socket = ZMQSocket(zmq_context, zmq.DEALER) socket.connect('ipc://zmqbug') class Cnt: diff --git a/zerorpc/context.py b/zerorpc/context.py index 7c53a36..8d8f5f0 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -32,7 +32,17 @@ class Context(zmq.Context): _instance = None + # Since pyzmq 13.0.0 implicit assignation is forbidden. Thankfully we are + # allowed to define our attributes here, and initialize them per instance + # later. + _middlewares = None + _hooks = None + _msg_id_base = None + _msg_id_counter = None + _msg_id_counter_stop = None + def __init__(self): + super(zmq.Context, self).__init__() self._middlewares = [] self._hooks = { 'resolve_endpoint': [], diff --git a/zerorpc/core.py b/zerorpc/core.py index 5e0b0eb..065791d 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -260,7 +260,7 @@ class Server(SocketBase, ServerBase): def __init__(self, methods=None, name=None, context=None, pool_size=None, heartbeat=5): - SocketBase.__init__(self, zmq.XREP, context) + SocketBase.__init__(self, zmq.ROUTER, context) if methods is None: methods = self @@ -278,7 +278,7 @@ class Client(SocketBase, ClientBase): def __init__(self, connect_to=None, context=None, timeout=30, heartbeat=5, passive_heartbeat=False): - SocketBase.__init__(self, zmq.XREQ, context=context) + SocketBase.__init__(self, zmq.DEALER, context=context) ClientBase.__init__(self, self._events, context, timeout, heartbeat, passive_heartbeat) if connect_to: diff --git a/zerorpc/events.py b/zerorpc/events.py index 7911151..8f1ee55 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -177,18 +177,21 @@ def __init__(self, zmq_socket_type, context=None): self._socket = zmq.Socket(self._context, zmq_socket_type) self._send = self._socket.send_multipart self._recv = self._socket.recv_multipart - if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.XREQ, zmq.XREP): + if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.DEALER, zmq.ROUTER): self._send = Sender(self._socket) - if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.XREQ, zmq.XREP): + if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER): self._recv = Receiver(self._socket) @property def recv_is_available(self): - return self._zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.XREQ, zmq.XREP) + return self._zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER) def __del__(self): - if not self._socket.closed: - self.close() + try: + if not self._socket.closed: + self.close() + except AttributeError: + pass def close(self): try: @@ -235,7 +238,7 @@ def emit_event(self, event, identity=None): if identity is not None: parts = list(identity) parts.extend(['', event.pack()]) - elif self._zmq_socket_type in (zmq.XREQ, zmq.XREP): + elif self._zmq_socket_type in (zmq.DEALER, zmq.ROUTER): parts = ('', event.pack()) else: parts = (event.pack(),) diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index 017b1f5..6bd9231 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -47,17 +47,21 @@ class Socket(_zmq.Socket): def __init__(self, context, socket_type): super(Socket, self).__init__(context, socket_type) on_state_changed_fd = self.getsockopt(_zmq.FD) - self._readable = gevent.event.Event() - self._writable = gevent.event.Event() + # Since pyzmq 13.0.0 implicit assignation is forbidden. Trying to define + # the attributes as a class member first do not work either because the + # setattr is a non-op! So we are doing it the ugly way. + self.__dict__["_readable"] = gevent.event.Event() + self.__dict__["_writable"] = gevent.event.Event() try: # gevent>=1.0 - self._state_event = gevent.hub.get_hub().loop.io( - on_state_changed_fd, gevent.core.READ) + self.__dict__["_state_event"] = gevent.hub.get_hub().loop.io( + on_state_changed_fd, gevent.core.READ) self._state_event.start(self._on_state_changed) except AttributeError: # gevent<1.0 - self._state_event = gevent.core.read_event(on_state_changed_fd, - self._on_state_changed, persist=True) + self.__dict__["_state_event"] = \ + gevent.core.read_event(on_state_changed_fd, + self._on_state_changed, persist=True) def _on_state_changed(self, event=None, _evtype=None): if self.closed: From 90f7c8eb23d52232ab5fd24ef090eecf89c90d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Wed, 6 Mar 2013 21:41:07 -0800 Subject: [PATCH 003/113] update test, to help reproduce bug in travis It seems that Travis has something special that makes some test crash pretty bad. Its really hard if not impossible to reproduce, for now I extended the heartbeat frequency of the tests to 2 seconds (to be longer than the maximum wait on the (py)ZMQ bug of missing events). I also used a gevent.Pool to forward exceptions. --- tests/test_buffered_channel.py | 79 +++++++++++++++++----------------- tests/test_heartbeat.py | 32 +++++++------- 2 files changed, 56 insertions(+), 55 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index ec67f2c..c1d298e 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -42,13 +42,13 @@ def test_close_server_bufchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() @@ -74,13 +74,13 @@ def test_close_client_bufchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() @@ -106,12 +106,12 @@ def test_heartbeat_can_open_channel_server_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) gevent.sleep(3) @@ -136,12 +136,12 @@ def test_heartbeat_can_open_channel_client_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) gevent.sleep(3) @@ -166,12 +166,12 @@ def test_do_some_req_rep(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) def client_do(): @@ -182,7 +182,8 @@ def client_do(): assert event.args == (x + x * x,) client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): for x in xrange(20): @@ -191,10 +192,9 @@ def server_do(): server_bufchan.emit('OK', (sum(event.args),)) server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() @@ -212,7 +212,7 @@ def test_do_some_req_rep_lost_server(): def client_do(): print 'running' client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) for x in xrange(10): client_bufchan.emit('add', (x, x * x)) @@ -224,12 +224,13 @@ def client_do(): event = client_bufchan.recv() client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) for x in xrange(10): event = server_bufchan.recv() @@ -237,10 +238,9 @@ def server_do(): server_bufchan.emit('OK', (sum(event.args),)) server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() @@ -257,7 +257,7 @@ def test_do_some_req_rep_lost_client(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) for x in xrange(10): @@ -267,12 +267,13 @@ def client_do(): assert event.args == (x + x * x,) client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) for x in xrange(10): @@ -284,10 +285,9 @@ def server_do(): event = server_bufchan.recv() server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() @@ -304,7 +304,7 @@ def test_do_some_req_rep_client_timeout(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) with assert_raises(zerorpc.TimeoutExpired): @@ -315,12 +315,13 @@ def client_do(): assert event.args == (x,) client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) with assert_raises(zerorpc.LostRemote): @@ -331,10 +332,10 @@ def server_do(): server_bufchan.emit('OK', event.args) server_bufchan.close() - server_task = gevent.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.spawn(server_do) + + coro_pool.join() client.close() server.close() @@ -354,12 +355,12 @@ def test_congestion_control_server_pushing(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) def client_do(): @@ -368,7 +369,8 @@ def client_do(): assert event.name == 'coucou' assert event.args == x - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): with assert_raises(CongestionError): @@ -383,12 +385,11 @@ def server_do(): for x in xrange(101, 200): server_bufchan.emit('coucou', x) # block until receiver is ready - server_task = gevent.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.spawn(server_do) + + coro_pool.join() client_bufchan.close() client.close() - server_task.get() server_bufchan.close() server.close() diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 9ba1c85..a03d504 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -42,12 +42,12 @@ def test_close_server_hbchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_hbchan.recv() gevent.sleep(3) @@ -72,12 +72,12 @@ def test_close_client_hbchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_hbchan.recv() gevent.sleep(3) @@ -102,11 +102,11 @@ def test_heartbeat_can_open_channel_server_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) gevent.sleep(3) print 'CLOSE SERVER SOCKET!!!' @@ -130,11 +130,11 @@ def test_heartbeat_can_open_channel_client_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) gevent.sleep(3) print 'CLOSE CLIENT SOCKET!!!' @@ -158,11 +158,11 @@ def test_do_some_req_rep(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) def client_do(): for x in xrange(20): @@ -202,7 +202,7 @@ def test_do_some_req_rep_lost_server(): def client_do(): print 'running' client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) for x in xrange(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() @@ -218,7 +218,7 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) for x in xrange(10): event = server_hbchan.recv() assert event.name == 'add' @@ -245,7 +245,7 @@ def test_do_some_req_rep_lost_client(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) for x in xrange(10): client_hbchan.emit('add', (x, x * x)) @@ -259,7 +259,7 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) for x in xrange(10): event = server_hbchan.recv() @@ -290,7 +290,7 @@ def test_do_some_req_rep_client_timeout(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) with assert_raises(zerorpc.TimeoutExpired): for x in xrange(10): @@ -305,7 +305,7 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) with assert_raises(zerorpc.LostRemote): for x in xrange(20): From 02ef63ed64f49ee3b14ff3c5607aafcfd9e55dd9 Mon Sep 17 00:00:00 2001 From: Louis Opter Date: Fri, 8 Mar 2013 15:30:51 -0800 Subject: [PATCH 004/113] "Less ugly" fix for context.py and clarify comments about pyzmq 13.0.0 --- zerorpc/context.py | 53 +++++++++++++++++++++++++++++++++++-------- zerorpc/gevent_zmq.py | 6 ++--- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/zerorpc/context.py b/zerorpc/context.py index 8d8f5f0..f489509 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -32,15 +32,6 @@ class Context(zmq.Context): _instance = None - # Since pyzmq 13.0.0 implicit assignation is forbidden. Thankfully we are - # allowed to define our attributes here, and initialize them per instance - # later. - _middlewares = None - _hooks = None - _msg_id_base = None - _msg_id_counter = None - _msg_id_counter_stop = None - def __init__(self): super(zmq.Context, self).__init__() self._middlewares = [] @@ -57,6 +48,50 @@ def __init__(self): } self._reset_msgid() + # NOTE: pyzmq 13.0.0 messed up with setattr (they turned it into a + # non-op) and you can't assign attributes normally anymore, hence the + # tricks with self.__dict__ here + + @property + def _middlewares(self): + return self.__dict__['_middlewares'] + + @_middlewares.setter + def _middlewares(self, value): + self.__dict__['_middlewares'] = value + + @property + def _hooks(self): + return self.__dict__['_hooks'] + + @_hooks.setter + def _hooks(self, value): + self.__dict__['_hooks'] = value + + @property + def _msg_id_base(self): + return self.__dict__['_msg_id_base'] + + @_msg_id_base.setter + def _msg_id_base(self, value): + self.__dict__['_msg_id_base'] = value + + @property + def _msg_id_counter(self): + return self.__dict__['_msg_id_counter'] + + @_msg_id_counter.setter + def _msg_id_counter(self, value): + self.__dict__['_msg_id_counter'] = value + + @property + def _msg_id_counter_stop(self): + return self.__dict__['_msg_id_counter_stop'] + + @_msg_id_counter_stop.setter + def _msg_id_counter_stop(self, value): + self.__dict__['_msg_id_counter_stop'] = value + @staticmethod def get_instance(): if Context._instance is None: diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index 6bd9231..d17e49a 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -47,9 +47,9 @@ class Socket(_zmq.Socket): def __init__(self, context, socket_type): super(Socket, self).__init__(context, socket_type) on_state_changed_fd = self.getsockopt(_zmq.FD) - # Since pyzmq 13.0.0 implicit assignation is forbidden. Trying to define - # the attributes as a class member first do not work either because the - # setattr is a non-op! So we are doing it the ugly way. + # NOTE: pyzmq 13.0.0 messed up with setattr (they turned it into a + # non-op) and you can't assign attributes normally anymore, hence the + # tricks with self.__dict__ here self.__dict__["_readable"] = gevent.event.Event() self.__dict__["_writable"] = gevent.event.Event() try: From dd30e2cb710733e2b9bbdff6791283219855400c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 9 Jun 2013 18:46:45 -0700 Subject: [PATCH 005/113] Retry on EINTR error For whatever reason ZMQ now raises ZMQErrors with an errno.EINTR code. We simply retry when they occur. fixes #62 --- zerorpc/gevent_zmq.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index d17e49a..401b2df 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -33,6 +33,7 @@ import gevent.event import gevent.core import sys +import errno class Context(_zmq.Context): @@ -104,7 +105,7 @@ def send(self, data, flags=0, copy=True, track=False): self._on_state_changed() return msg except _zmq.ZMQError, e: - if e.errno != _zmq.EAGAIN: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._writable.clear() # The following sleep(0) force gevent to switch out to another @@ -140,7 +141,7 @@ def recv(self, flags=0, copy=True, track=False): self._on_state_changed() return msg except _zmq.ZMQError, e: - if e.errno != _zmq.EAGAIN: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._readable.clear() # The following sleep(0) force gevent to switch out to another From 2eeac40c337131012d922892e370f082e2f2f95a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 9 Jun 2013 21:20:38 -0700 Subject: [PATCH 006/113] Fix a broken unit test For who is intersted by the resolution: - a channel cannot be considered opened until the endpoint answered - the code was sending 2 events in a row... - the first event will open the channel - but if the second event arrive "too fast", the channel wont be "fully opened" The solution: - wrap the channel in a BufferedChannel, which automatically prevent sending more than one message until the channel is opened fully. --- tests/test_wrapped_events.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py index 64024f2..f363d37 100644 --- a/tests/test_wrapped_events.py +++ b/tests/test_wrapped_events.py @@ -24,6 +24,7 @@ import random +import gevent from zerorpc import zmq import zerorpc @@ -74,11 +75,14 @@ def test_multiple_sub_events(): client_channel1 = client.channel() client_channel_events1 = zerorpc.WrappedEvents(client_channel1) - client_channel2 = client.channel() + client_channel2 = zerorpc.BufferedChannel(client.channel()) client_channel_events2 = zerorpc.WrappedEvents(client_channel2) - client_channel_events1.emit('coucou1', 43) - client_channel_events2.emit('coucou2', 44) - client_channel_events2.emit('another', 42) + + def emitstuff(): + client_channel_events1.emit('coucou1', 43) + client_channel_events2.emit('coucou2', 44) + client_channel_events2.emit('another', 42) + gevent.spawn(emitstuff) event = server.recv() print event @@ -89,7 +93,7 @@ def test_multiple_sub_events(): server_channel = server.channel(event) server_channel_events = zerorpc.WrappedEvents(server_channel) event = server_channel_events.recv() - print event + print 'ch1:', event assert event.name == 'coucou1' assert event.args == 43 @@ -100,14 +104,16 @@ def test_multiple_sub_events(): subevent = event.args print 'subevent:', subevent server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) + + server_channel_events = zerorpc.BufferedChannel(server_channel) + server_channel_events = zerorpc.WrappedEvents(server_channel_events) event = server_channel_events.recv() - print event + print 'ch2:', event assert event.name == 'coucou2' assert event.args == 44 event = server_channel_events.recv() - print event + print 'ch2:', event assert event.name == 'another' assert event.args == 42 From 076286822940ba266bff0834eace423388b079c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 9 Jun 2013 21:23:47 -0700 Subject: [PATCH 007/113] bump version to v0.4.2 --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index a6db92a..b32d4a0 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.4.1' +__version__ = '0.4.2' __author__ = 'dotCloud, Inc.' __license__ = 'MIT' __copyright__ = 'Copyright 2012 dotCloud, Inc.' From ad4bd9aedacf304070aa2f77c95c2034ba3a8ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 9 Jun 2013 23:44:30 -0700 Subject: [PATCH 008/113] EINTR, EINTR eveywhere! for some reason, zmq thorw EINTR on getsockopt calls now... ref #62 --- zerorpc/gevent_zmq.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index 401b2df..fdef37b 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -70,7 +70,14 @@ def _on_state_changed(self, event=None, _evtype=None): self._readable.set() return - events = self.getsockopt(_zmq.EVENTS) + while True: + try: + events = self.getsockopt(_zmq.EVENTS) + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise + if events & _zmq.POLLOUT: self._writable.set() if events & _zmq.POLLIN: From 3fd2da8da478229dc466053da0301f723543d91f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Mon, 10 Jun 2013 20:23:47 -0700 Subject: [PATCH 009/113] bump version to v0.4.3 --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index b32d4a0..5df42b7 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.4.2' +__version__ = '0.4.3' __author__ = 'dotCloud, Inc.' __license__ = 'MIT' __copyright__ = 'Copyright 2012 dotCloud, Inc.' From 528193b2b28c8defa1d58718db8dfda52d2d9712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 12 Jun 2013 15:53:08 -0700 Subject: [PATCH 010/113] the latest zerorpc code needs at least pyzmq 13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4acc49..7d74199 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ requirements = [ 'gevent', 'msgpack-python', - 'pyzmq>=2.2.0.1' + 'pyzmq>=13' ] if sys.version_info < (2, 7): requirements.append('argparse') From 451d8fd4f8400d2fdda218849b46f1fd1a947577 Mon Sep 17 00:00:00 2001 From: Arnon Yaari Date: Wed, 10 Jul 2013 22:51:06 +0300 Subject: [PATCH 011/113] fix tests event args were expected to be tuples, but they are lists --- tests/test_buffered_channel.py | 8 ++++---- tests/test_channel.py | 14 +++++++------- tests/test_events.py | 14 +++++++------- tests/test_heartbeat.py | 8 ++++---- tests/test_reqstream.py | 2 +- tests/test_server.py | 4 ++-- tests/test_wrapped_events.py | 6 +++--- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index c1d298e..f3d5e6f 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -179,7 +179,7 @@ def client_do(): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert event.args == [x + x * x] client_bufchan.close() coro_pool = gevent.pool.Pool() @@ -218,7 +218,7 @@ def client_do(): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert event.args == [x + x * x] client_bufchan.emit('add', (x, x * x)) with assert_raises(zerorpc.LostRemote): event = client_bufchan.recv() @@ -264,7 +264,7 @@ def client_do(): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert event.args == [x + x * x] client_bufchan.close() coro_pool = gevent.pool.Pool() @@ -312,7 +312,7 @@ def client_do(): client_bufchan.emit('sleep', (x,)) event = client_bufchan.recv(timeout=3) assert event.name == 'OK' - assert event.args == (x,) + assert event.args == [x] client_bufchan.close() coro_pool = gevent.pool.Pool() diff --git a/tests/test_channel.py b/tests/test_channel.py index d58d862..76adc08 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -43,14 +43,14 @@ def test_events_channel_client_side(): event = server.recv() print event - assert event.args == (42,) + assert event.args == [42] assert event.header.get('zmqid', None) is not None server.emit('someanswer', (21,), xheader=dict(response_to=event.header['message_id'], zmqid=event.header['zmqid'])) event = client_channel.recv() - assert event.args == (21,) + assert event.args == [21] def test_events_channel_client_side_server_send_many(): @@ -68,7 +68,7 @@ def test_events_channel_client_side_server_send_many(): event = server.recv() print event - assert event.args == (10,) + assert event.args == [10] assert event.header.get('zmqid', None) is not None for x in xrange(10): @@ -77,7 +77,7 @@ def test_events_channel_client_side_server_send_many(): zmqid=event.header['zmqid'])) for x in xrange(10): event = client_channel.recv() - assert event.args == (x,) + assert event.args == [x] def test_events_channel_both_side(): @@ -95,20 +95,20 @@ def test_events_channel_both_side(): event = server.recv() print event - assert event.args == (42,) + assert event.args == [42] assert event.name == 'openthat' server_channel = server.channel(event) server_channel.emit('test', (21,)) event = client_channel.recv() - assert event.args == (21,) + assert event.args == [21] assert event.name == 'test' server_channel.emit('test', (22,)) event = client_channel.recv() - assert event.args == (22,) + assert event.args == [22] assert event.name == 'test' server_events.close() diff --git a/tests/test_events.py b/tests/test_events.py index 537adfc..2f0ae01 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -73,7 +73,7 @@ def test_event(): assert unpacked.name == 'mylittleevent4' assert unpacked.header['message_id'] == 3 - assert unpacked.args == ('b', 21) + assert unpacked.args == ['b', 21] event = zerorpc.Event('mylittleevent5', ('c', 24, True), header={'lol': 'rofl'}, context=None) @@ -104,7 +104,7 @@ def test_events_req_rep(): event = server.recv() print event assert event.name == 'myevent' - assert event.args == ('arg1',) + assert event.args == ['arg1'] def test_events_req_rep2(): @@ -120,13 +120,13 @@ def test_events_req_rep2(): event = server.recv() print event assert event.name == 'myevent' + str(i) - assert event.args == (i,) + assert event.args == [i] server.emit('answser' + str(i * 2), (i * 2,)) event = client.recv() print event assert event.name == 'answser' + str(i * 2) - assert event.args == (i * 2,) + assert event.args == [i * 2] def test_events_dealer_router(): @@ -142,14 +142,14 @@ def test_events_dealer_router(): event = server.recv() print event assert event.name == 'myevent' + str(i) - assert event.args == (i,) + assert event.args == [i] server.emit('answser' + str(i * 2), (i * 2,), xheader=dict(zmqid=event.header['zmqid'])) event = client.recv() print event assert event.name == 'answser' + str(i * 2) - assert event.args == (i * 2,) + assert event.args == [i * 2] def test_events_push_pull(): @@ -167,4 +167,4 @@ def test_events_push_pull(): event = server.recv() print event assert event.name == 'myevent' - assert event.args == (x,) + assert event.args == [x] diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index a03d504..858f2f4 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -169,7 +169,7 @@ def client_do(): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert event.args == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -207,7 +207,7 @@ def client_do(): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert event.args == [x + x * x] client_hbchan.emit('add', (x, x * x)) with assert_raises(zerorpc.LostRemote): event = client_hbchan.recv() @@ -251,7 +251,7 @@ def client_do(): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert event.args == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -297,7 +297,7 @@ def client_do(): client_hbchan.emit('sleep', (x,)) event = client_hbchan.recv(timeout=3) assert event.name == 'OK' - assert event.args == (x,) + assert event.args == [x] client_hbchan.close() client_task = gevent.spawn(client_do) diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index 6b79ddb..0819be4 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -50,7 +50,7 @@ def xrange(self, max): client.connect(endpoint) r = client.range(10) - assert r == tuple(range(10)) + assert r == list(range(10)) r = client.xrange(10) assert getattr(r, 'next', None) is not None diff --git a/tests/test_server.py b/tests/test_server.py index e6e2ddf..9743ce3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -53,13 +53,13 @@ def add(self, a, b): client_channel = client.channel() client_channel.emit('lolita', tuple()) event = client_channel.recv() - assert event.args == (42,) + assert event.args == [42] client_channel.close() client_channel = client.channel() client_channel.emit('add', (1, 2)) event = client_channel.recv() - assert event.args == (3,) + assert event.args == [3] client_channel.close() srv.stop() diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py index f363d37..195959b 100644 --- a/tests/test_wrapped_events.py +++ b/tests/test_wrapped_events.py @@ -47,7 +47,7 @@ def test_sub_events(): event = server.recv() print event - assert type(event.args) is tuple + assert type(event.args) is list assert event.name == 'w' subevent = event.args print 'subevent:', subevent @@ -86,7 +86,7 @@ def emitstuff(): event = server.recv() print event - assert type(event.args) is tuple + assert type(event.args) is list assert event.name == 'w' subevent = event.args print 'subevent:', subevent @@ -99,7 +99,7 @@ def emitstuff(): event = server.recv() print event - assert type(event.args) is tuple + assert type(event.args) is list assert event.name == 'w' subevent = event.args print 'subevent:', subevent From 2f75bccb4f22abaf3adc0e32d2fb47ff5149d0dd Mon Sep 17 00:00:00 2001 From: Arnon Yaari Date: Wed, 10 Jul 2013 22:52:36 +0300 Subject: [PATCH 012/113] use logging instead of printing to stderr --- zerorpc/channel.py | 25 +++++++++++++++---------- zerorpc/core.py | 8 ++++++-- zerorpc/gevent_zmq.py | 11 +++++++---- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 199900b..744bad5 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -32,6 +32,11 @@ from .exceptions import TimeoutExpired +from logging import getLogger + +logger = getLogger(__name__) + + class ChannelMultiplexer(object): def __init__(self, events, ignore_broadcast=False): @@ -76,9 +81,9 @@ def _channel_dispatcher(self): try: event = self._events.recv() except Exception as e: - print >> sys.stderr, \ - 'zerorpc.ChannelMultiplexer,', \ - 'ignoring error on recv: {0}'.format(e) + logger.error( \ + 'zerorpc.ChannelMultiplexer, ' + \ + 'ignoring error on recv: {0}'.format(e)) continue channel_id = event.header.get('response_to', None) @@ -91,10 +96,10 @@ def _channel_dispatcher(self): queue = self._broadcast_queue if queue is None: - print >> sys.stderr, \ - 'zerorpc.ChannelMultiplexer,', \ - 'unable to route event:', \ - event.__str__(ignore_args=True) + logger.error( \ + 'zerorpc.ChannelMultiplexer, ' + \ + 'unable to route event: ' + \ + event.__str__(ignore_args=True)) else: queue.put(event) @@ -210,9 +215,9 @@ def _recver(self): try: self._remote_queue_open_slots += int(event.args[0]) except Exception as e: - print >> sys.stderr, \ - 'gevent_zerorpc.BufferedChannel._recver,', \ - 'exception:', e + logger.error( \ + 'gevent_zerorpc.BufferedChannel._recver, ' + \ + 'exception: ' + repr(e)) if self._remote_queue_open_slots > 0: self._remote_can_recv.set() elif self._input_queue.qsize() == self._input_queue_size: diff --git a/zerorpc/core.py b/zerorpc/core.py index 065791d..a4d0f82 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -39,6 +39,10 @@ from .context import Context from .decorators import DecoratorBase, rep import patterns +from logging import getLogger + +logger = getLogger(__name__) + class ServerBase(object): @@ -118,7 +122,7 @@ def __call__(self, method, *args): return self._methods[method](*args) def _print_traceback(self, protocol_v1, exc_infos): - traceback.print_exception(*exc_infos, file=sys.stderr) + logger.exception('') exc_type, exc_value, exc_traceback = exc_infos if protocol_v1: @@ -337,7 +341,7 @@ def _receiver(self): except Exception: exc_infos = sys.exc_info() try: - traceback.print_exception(*exc_infos, file=sys.stderr) + logger.exception('') self._context.hook_server_inspect_exception(event, None, exc_infos) finally: del exc_infos diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index fdef37b..6974d64 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -34,6 +34,9 @@ import gevent.core import sys import errno +from logging import getLogger + +logger = getLogger(__name__) class Context(_zmq.Context): @@ -125,8 +128,8 @@ def send(self, data, flags=0, copy=True, track=False): gevent.sleep(0) while not self._writable.wait(timeout=1): if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: - print>>sys.stderr, "/!\\ gevent_zeromq BUG /!\\ " \ - "catching up after missing event (SEND) /!\\" + logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ + "catching up after missing event (SEND) /!\\") break def recv(self, flags=0, copy=True, track=False): @@ -161,6 +164,6 @@ def recv(self, flags=0, copy=True, track=False): gevent.sleep(0) while not self._readable.wait(timeout=1): if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: - print>>sys.stderr, "/!\\ gevent_zeromq BUG /!\\ " \ - "catching up after missing event (RECV) /!\\" + logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ + "catching up after missing event (RECV) /!\\") break From b8b7f9267a0c868e5a5b9b2832e36f47a63e0fb3 Mon Sep 17 00:00:00 2001 From: Guy Rozendorn Date: Tue, 8 Oct 2013 14:54:39 +0300 Subject: [PATCH 013/113] Issue #74 retry connect and getsockopt on EINTR Conflicts: zerorpc/gevent_zmq.py --- zerorpc/gevent_zmq.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index fdef37b..9c043d0 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -93,6 +93,14 @@ def close(self): self._state_event.cancel() super(Socket, self).close() + def connect(self, *args, **kwargs): + while True: + try: + return super(Socket, self).connect(*args, **kwargs) + except _zmq.ZMQError, e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise + def send(self, data, flags=0, copy=True, track=False): if flags & _zmq.NOBLOCK: return super(Socket, self).send(data, flags, copy, track) @@ -124,10 +132,14 @@ def send(self, data, flags=0, copy=True, track=False): # the same CPU load, you get a better throughput (roughly 18.75%). gevent.sleep(0) while not self._writable.wait(timeout=1): - if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: - print>>sys.stderr, "/!\\ gevent_zeromq BUG /!\\ " \ - "catching up after missing event (SEND) /!\\" - break + try: + if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: + print>>sys.stderr, "/!\\ gevent_zeromq BUG /!\\ " \ + "catching up after missing event (SEND) /!\\" + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise def recv(self, flags=0, copy=True, track=False): if flags & _zmq.NOBLOCK: @@ -160,7 +172,10 @@ def recv(self, flags=0, copy=True, track=False): # the same CPU load, you get a better throughput (roughly 18.75%). gevent.sleep(0) while not self._readable.wait(timeout=1): - if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: + try: print>>sys.stderr, "/!\\ gevent_zeromq BUG /!\\ " \ - "catching up after missing event (RECV) /!\\" - break + "catching up after missing event (RECV) /!\\") + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise From aa734c76030c4360fabd3926a2f05e14675065b2 Mon Sep 17 00:00:00 2001 From: Arnon Yaari Date: Thu, 24 Oct 2013 18:57:27 +0300 Subject: [PATCH 014/113] tests - support event.args as both tuple and list --- tests/test_buffered_channel.py | 8 ++++---- tests/test_channel.py | 14 +++++++------- tests/test_events.py | 14 +++++++------- tests/test_heartbeat.py | 8 ++++---- tests/test_reqstream.py | 2 +- tests/test_server.py | 4 ++-- tests/test_wrapped_events.py | 6 +++--- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index f3d5e6f..6df5d38 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -179,7 +179,7 @@ def client_do(): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == [x + x * x] + assert list(event.args) == [x + x * x] client_bufchan.close() coro_pool = gevent.pool.Pool() @@ -218,7 +218,7 @@ def client_do(): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == [x + x * x] + assert list(event.args) == [x + x * x] client_bufchan.emit('add', (x, x * x)) with assert_raises(zerorpc.LostRemote): event = client_bufchan.recv() @@ -264,7 +264,7 @@ def client_do(): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == [x + x * x] + assert list(event.args) == [x + x * x] client_bufchan.close() coro_pool = gevent.pool.Pool() @@ -312,7 +312,7 @@ def client_do(): client_bufchan.emit('sleep', (x,)) event = client_bufchan.recv(timeout=3) assert event.name == 'OK' - assert event.args == [x] + assert list(event.args) == [x] client_bufchan.close() coro_pool = gevent.pool.Pool() diff --git a/tests/test_channel.py b/tests/test_channel.py index 76adc08..ed8367e 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -43,14 +43,14 @@ def test_events_channel_client_side(): event = server.recv() print event - assert event.args == [42] + assert list(event.args) == [42] assert event.header.get('zmqid', None) is not None server.emit('someanswer', (21,), xheader=dict(response_to=event.header['message_id'], zmqid=event.header['zmqid'])) event = client_channel.recv() - assert event.args == [21] + assert list(event.args) == [21] def test_events_channel_client_side_server_send_many(): @@ -68,7 +68,7 @@ def test_events_channel_client_side_server_send_many(): event = server.recv() print event - assert event.args == [10] + assert list(event.args) == [10] assert event.header.get('zmqid', None) is not None for x in xrange(10): @@ -77,7 +77,7 @@ def test_events_channel_client_side_server_send_many(): zmqid=event.header['zmqid'])) for x in xrange(10): event = client_channel.recv() - assert event.args == [x] + assert list(event.args) == [x] def test_events_channel_both_side(): @@ -95,20 +95,20 @@ def test_events_channel_both_side(): event = server.recv() print event - assert event.args == [42] + assert list(event.args) == [42] assert event.name == 'openthat' server_channel = server.channel(event) server_channel.emit('test', (21,)) event = client_channel.recv() - assert event.args == [21] + assert list(event.args) == [21] assert event.name == 'test' server_channel.emit('test', (22,)) event = client_channel.recv() - assert event.args == [22] + assert list(event.args) == [22] assert event.name == 'test' server_events.close() diff --git a/tests/test_events.py b/tests/test_events.py index 2f0ae01..19ca208 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -73,7 +73,7 @@ def test_event(): assert unpacked.name == 'mylittleevent4' assert unpacked.header['message_id'] == 3 - assert unpacked.args == ['b', 21] + assert list(unpacked.args) == ['b', 21] event = zerorpc.Event('mylittleevent5', ('c', 24, True), header={'lol': 'rofl'}, context=None) @@ -104,7 +104,7 @@ def test_events_req_rep(): event = server.recv() print event assert event.name == 'myevent' - assert event.args == ['arg1'] + assert list(event.args) == ['arg1'] def test_events_req_rep2(): @@ -120,13 +120,13 @@ def test_events_req_rep2(): event = server.recv() print event assert event.name == 'myevent' + str(i) - assert event.args == [i] + assert list(event.args) == [i] server.emit('answser' + str(i * 2), (i * 2,)) event = client.recv() print event assert event.name == 'answser' + str(i * 2) - assert event.args == [i * 2] + assert list(event.args) == [i * 2] def test_events_dealer_router(): @@ -142,14 +142,14 @@ def test_events_dealer_router(): event = server.recv() print event assert event.name == 'myevent' + str(i) - assert event.args == [i] + assert list(event.args) == [i] server.emit('answser' + str(i * 2), (i * 2,), xheader=dict(zmqid=event.header['zmqid'])) event = client.recv() print event assert event.name == 'answser' + str(i * 2) - assert event.args == [i * 2] + assert list(event.args) == [i * 2] def test_events_push_pull(): @@ -167,4 +167,4 @@ def test_events_push_pull(): event = server.recv() print event assert event.name == 'myevent' - assert event.args == [x] + assert list(event.args) == [x] diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 858f2f4..36bd01d 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -169,7 +169,7 @@ def client_do(): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == [x + x * x] + assert list(event.args) == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -207,7 +207,7 @@ def client_do(): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == [x + x * x] + assert list(event.args) == [x + x * x] client_hbchan.emit('add', (x, x * x)) with assert_raises(zerorpc.LostRemote): event = client_hbchan.recv() @@ -251,7 +251,7 @@ def client_do(): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == [x + x * x] + assert list(event.args) == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -297,7 +297,7 @@ def client_do(): client_hbchan.emit('sleep', (x,)) event = client_hbchan.recv(timeout=3) assert event.name == 'OK' - assert event.args == [x] + assert list(event.args) == [x] client_hbchan.close() client_task = gevent.spawn(client_do) diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index 0819be4..ee30423 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -50,7 +50,7 @@ def xrange(self, max): client.connect(endpoint) r = client.range(10) - assert r == list(range(10)) + assert list(r) == list(range(10)) r = client.xrange(10) assert getattr(r, 'next', None) is not None diff --git a/tests/test_server.py b/tests/test_server.py index 9743ce3..904d87c 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -53,13 +53,13 @@ def add(self, a, b): client_channel = client.channel() client_channel.emit('lolita', tuple()) event = client_channel.recv() - assert event.args == [42] + assert list(event.args) == [42] client_channel.close() client_channel = client.channel() client_channel.emit('add', (1, 2)) event = client_channel.recv() - assert event.args == [3] + assert list(event.args) == [3] client_channel.close() srv.stop() diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py index 195959b..43c42fe 100644 --- a/tests/test_wrapped_events.py +++ b/tests/test_wrapped_events.py @@ -47,7 +47,7 @@ def test_sub_events(): event = server.recv() print event - assert type(event.args) is list + assert isinstance(event.args, (list, tuple)) assert event.name == 'w' subevent = event.args print 'subevent:', subevent @@ -86,7 +86,7 @@ def emitstuff(): event = server.recv() print event - assert type(event.args) is list + assert isinstance(event.args, (list, tuple)) assert event.name == 'w' subevent = event.args print 'subevent:', subevent @@ -99,7 +99,7 @@ def emitstuff(): event = server.recv() print event - assert type(event.args) is list + assert isinstance(event.args, (list, tuple)) assert event.name == 'w' subevent = event.args print 'subevent:', subevent From a6e8f1816c25991ee42105067b47a0308bb77510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Thu, 24 Oct 2013 11:34:36 -0700 Subject: [PATCH 015/113] Fix messed up indentation --- zerorpc/gevent_zmq.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index b56a0f4..edda633 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -137,8 +137,8 @@ def send(self, data, flags=0, copy=True, track=False): while not self._writable.wait(timeout=1): try: if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: - logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ - "catching up after missing event (SEND) /!\\") + logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ + "catching up after missing event (SEND) /!\\") break except ZMQError as e: if e.errno not in (_zmq.EAGAIN, errno.EINTR): @@ -176,10 +176,10 @@ def recv(self, flags=0, copy=True, track=False): gevent.sleep(0) while not self._readable.wait(timeout=1): try: - if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: - logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ - "catching up after missing event (RECV) /!\\") - break + if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: + logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ + "catching up after missing event (RECV) /!\\") + break except ZMQError as e: if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise From 60ada1e3f6509e409b2ba1610e73011a3e850a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 30 Oct 2013 14:51:40 -0700 Subject: [PATCH 016/113] Pin pyzmq==13.1.0 because 14 BREAKS ALL THE THINGS. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7d74199..4e566a9 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ requirements = [ 'gevent', 'msgpack-python', - 'pyzmq>=13' + 'pyzmq==13.1.0' ] if sys.version_info < (2, 7): requirements.append('argparse') From 57c1d9eb55d9166dacb673caaf6988ded045c95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 30 Oct 2013 14:53:08 -0700 Subject: [PATCH 017/113] bump version to v0.4.4 --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index 5df42b7..76a6ecf 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.4.3' +__version__ = '0.4.4' __author__ = 'dotCloud, Inc.' __license__ = 'MIT' __copyright__ = 'Copyright 2012 dotCloud, Inc.' From 55732b849abeaab29fdda3360b49eb61e8a7ecf4 Mon Sep 17 00:00:00 2001 From: JJ Geewax Date: Mon, 11 Nov 2013 11:28:09 -0500 Subject: [PATCH 018/113] Additional fixes for Issue #41 Issue #41 was about an `zmq.core.error.ZMQError: Invalid argument` error when trying to connect to `tcp://*:4242`, however I ran into the same bug in Ubuntu 13.04 connecting to `tcp://:4242`. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index cdcf386..16a7022 100644 --- a/README.rst +++ b/README.rst @@ -186,11 +186,11 @@ Let's save this code to *cooler.py* and run it:: Now, in another terminal, let's try connecting to our awesome zeroservice:: - $ zerorpc -j tcp://:4242 add_42 1 + $ zerorpc -j tcp://localhost:4242 add_42 1 43 - $ zerorpc tcp://:4242 add_man 'I own a mint-condition Volkswagen Golf' + $ zerorpc tcp://localhost:4242 add_man 'I own a mint-condition Volkswagen Golf' "I own a mint-condition Volkswagen Golf, man!" - $ zerorpc tcp://:4242 boat 'I own a mint-condition Volkswagen Golf, man!' + $ zerorpc tcp://localhost:4242 boat 'I own a mint-condition Volkswagen Golf, man!' "I'm on a boat!" From 3778e0a1f675f2060ea4140b2858bb36b4999e5d Mon Sep 17 00:00:00 2001 From: Paulo SantAnna Date: Fri, 22 Nov 2013 13:11:31 -0800 Subject: [PATCH 019/113] Fixed deprecation warnings: gevent.queue.Queue(0) now means infinity queue, changing to Channel instead. gevent.coros module deprecated in favor of gevent.lock --- .gitignore | 1 + zerorpc/channel.py | 2 +- zerorpc/core.py | 2 +- zerorpc/events.py | 6 +++--- zerorpc/heartbeat.py | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 58311d7..8aa21a8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ *.egg *.egg-info .tox +.idea diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 744bad5..e304bbc 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -28,7 +28,7 @@ import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock from .exceptions import TimeoutExpired diff --git a/zerorpc/core.py b/zerorpc/core.py index a4d0f82..2be8067 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -29,7 +29,7 @@ import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock import gevent_zmq as zmq from .exceptions import TimeoutExpired, RemoteError, LostRemote diff --git a/zerorpc/events.py b/zerorpc/events.py index 8f1ee55..e4d79ee 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -28,7 +28,7 @@ import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock import gevent_zmq as zmq @@ -39,7 +39,7 @@ class Sender(object): def __init__(self, socket): self._socket = socket - self._send_queue = gevent.queue.Queue(maxsize=0) + self._send_queue = gevent.queue.Channel() self._send_task = gevent.spawn(self._sender) def __del__(self): @@ -72,7 +72,7 @@ class Receiver(object): def __init__(self, socket): self._socket = socket - self._recv_queue = gevent.queue.Queue(maxsize=0) + self._recv_queue = gevent.queue.Channel() self._recv_task = gevent.spawn(self._recver) def __del__(self): diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index e7af22f..92421af 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -28,7 +28,7 @@ import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock from .exceptions import * @@ -38,7 +38,7 @@ class HeartBeatOnChannel(object): def __init__(self, channel, freq=5, passive=False): self._channel = channel self._heartbeat_freq = freq - self._input_queue = gevent.queue.Queue(maxsize=0) + self._input_queue = gevent.queue.Channel() self._remote_last_hb = None self._lost_remote = False self._recv_task = gevent.spawn(self._recver) From 49899c8f9d767fb010a56722e21dbed5e93c1cd4 Mon Sep 17 00:00:00 2001 From: Paulo SantAnna Date: Mon, 25 Nov 2013 12:31:31 -0800 Subject: [PATCH 020/113] replaced localtime example as it does not return a serializable response. --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 16a7022..b44fe01 100644 --- a/README.rst +++ b/README.rst @@ -108,15 +108,15 @@ your server to act as a kind of worker, and connect to a hub or queue which will dispatch requests. You can achieve this by swapping "--bind" and "--connect":: - $ zerorpc --bind tcp://*:1234 localtime + $ zerorpc --bind tcp://*:1234 strftime %Y/%m/%d -We now have "something" wanting to call the "localtime" function, and waiting +We now have "something" wanting to call the "strftime" function, and waiting for a worker to connect to it. Let's start the worker:: - $ zerorpc --server tcp://*:1234 time + $ zerorpc --server tcp://127.0.0.1:1234 time The worker will connect to the listening client and ask him "what should I -do?"; the client will send the "localtime" function call; the worker will +do?"; the client will send the "strftime" function call; the worker will execute it and return the result. The first program will display the local time and exit. The worker will remain running. From 09b828325239f36f5c4f2dbe41f70b70a1a1747b Mon Sep 17 00:00:00 2001 From: Paulo SantAnna Date: Tue, 26 Nov 2013 15:40:58 -0800 Subject: [PATCH 021/113] Made tests backward compatible with python 2.6 and fixed some more deprecation warnings --- tests/test_buffered_channel.py | 115 ++++++++++++++++++++++++--------- tests/test_client_async.py | 11 +++- tests/test_heartbeat.py | 85 +++++++++++++++++------- tests/test_middleware.py | 12 ++-- tests/test_server.py | 26 ++++++-- 5 files changed, 181 insertions(+), 68 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index 6df5d38..f55993c 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -25,6 +25,7 @@ from nose.tools import assert_raises import gevent +import sys from zerorpc import zmq import zerorpc @@ -55,8 +56,11 @@ def test_close_server_bufchan(): gevent.sleep(3) print 'CLOSE SERVER SOCKET!!!' server_bufchan.close() - with assert_raises(zerorpc.LostRemote): - client_bufchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_bufchan.recv() print 'CLIENT LOST SERVER :)' client_bufchan.close() server.close() @@ -87,8 +91,11 @@ def test_close_client_bufchan(): gevent.sleep(3) print 'CLOSE CLIENT SOCKET!!!' client_bufchan.close() - with assert_raises(zerorpc.LostRemote): - server_bufchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_bufchan.recv() print 'SERVER LOST CLIENT :)' server_bufchan.close() server.close() @@ -117,8 +124,11 @@ def test_heartbeat_can_open_channel_server_close(): gevent.sleep(3) print 'CLOSE SERVER SOCKET!!!' server_bufchan.close() - with assert_raises(zerorpc.LostRemote): - client_bufchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_bufchan.recv() print 'CLIENT LOST SERVER :)' client_bufchan.close() server.close() @@ -148,8 +158,11 @@ def test_heartbeat_can_open_channel_client_close(): print 'CLOSE CLIENT SOCKET!!!' client_bufchan.close() client.close() - with assert_raises(zerorpc.LostRemote): - server_bufchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_bufchan.recv() print 'SERVER LOST CLIENT :)' server_bufchan.close() server.close() @@ -220,8 +233,11 @@ def client_do(): assert event.name == 'OK' assert list(event.args) == [x + x * x] client_bufchan.emit('add', (x, x * x)) - with assert_raises(zerorpc.LostRemote): - event = client_bufchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_bufchan.recv() client_bufchan.close() coro_pool = gevent.pool.Pool() @@ -281,8 +297,11 @@ def server_do(): assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) - with assert_raises(zerorpc.LostRemote): - event = server_bufchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, server_bufchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + server_bufchan.recv() server_bufchan.close() coro_pool.spawn(server_do) @@ -307,12 +326,21 @@ def client_do(): client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): - client_bufchan.emit('sleep', (x,)) - event = client_bufchan.recv(timeout=3) - assert event.name == 'OK' - assert list(event.args) == [x] + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in xrange(10): + client_bufchan.emit('sleep', (x,)) + event = client_bufchan.recv(timeout=3) + assert event.name == 'OK' + assert list(event.args) == [x] + assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with assert_raises(zerorpc.TimeoutExpired): + for x in xrange(10): + client_bufchan.emit('sleep', (x,)) + event = client_bufchan.recv(timeout=3) + assert event.name == 'OK' + assert list(event.args) == [x] client_bufchan.close() coro_pool = gevent.pool.Pool() @@ -324,12 +352,21 @@ def server_do(): server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - with assert_raises(zerorpc.LostRemote): - for x in xrange(20): - event = server_bufchan.recv() - assert event.name == 'sleep' - gevent.sleep(event.args[0]) - server_bufchan.emit('OK', event.args) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in xrange(20): + event = server_bufchan.recv() + assert event.name == 'sleep' + gevent.sleep(event.args[0]) + server_bufchan.emit('OK', event.args) + assert_raises(zerorpc.LostRemote, _do_with_assert_raises) + else: + with assert_raises(zerorpc.LostRemote): + for x in xrange(20): + event = server_bufchan.recv() + assert event.name == 'sleep' + gevent.sleep(event.args[0]) + server_bufchan.emit('OK', event.args) server_bufchan.close() @@ -373,15 +410,29 @@ def client_do(): coro_pool.spawn(client_do) def server_do(): - with assert_raises(CongestionError): - for x in xrange(200): - if server_bufchan.emit('coucou', x, block=False) == False: - raise CongestionError() # will fail when x == 1 + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in xrange(200): + if not server_bufchan.emit('coucou', x, block=False): + raise CongestionError() # will fail when x == 1 + assert_raises(CongestionError, _do_with_assert_raises) + else: + with assert_raises(CongestionError): + for x in xrange(200): + if not server_bufchan.emit('coucou', x, block=False): + raise CongestionError() # will fail when x == 1 server_bufchan.emit('coucou', 1) # block until receiver is ready - with assert_raises(CongestionError): - for x in xrange(2, 200): - if server_bufchan.emit('coucou', x, block=False) == False: - raise CongestionError() # will fail when x == 100 + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in xrange(2, 200): + if not server_bufchan.emit('coucou', x, block=False): + raise CongestionError() # will fail when x == 100 + assert_raises(CongestionError, _do_with_assert_raises) + else: + with assert_raises(CongestionError): + for x in xrange(2, 200): + if not server_bufchan.emit('coucou', x, block=False): + raise CongestionError() # will fail when x == 100 for x in xrange(101, 200): server_bufchan.emit('coucou', x) # block until receiver is ready diff --git a/tests/test_client_async.py b/tests/test_client_async.py index c939101..76e0abe 100644 --- a/tests/test_client_async.py +++ b/tests/test_client_async.py @@ -25,6 +25,7 @@ from nose.tools import assert_raises import gevent +import sys from zerorpc import zmq import zerorpc @@ -51,8 +52,14 @@ def add(self, a, b): client.connect(endpoint) async_result = client.add(1, 4, async=True) - with assert_raises(zerorpc.TimeoutExpired): - print async_result.get() + + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print async_result.get() + assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with assert_raises(zerorpc.TimeoutExpired): + print async_result.get() client.close() srv.close() diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 36bd01d..a4489e1 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -25,6 +25,7 @@ from nose.tools import assert_raises import gevent +import sys from zerorpc import zmq import zerorpc @@ -53,8 +54,11 @@ def test_close_server_hbchan(): gevent.sleep(3) print 'CLOSE SERVER SOCKET!!!' server_hbchan.close() - with assert_raises(zerorpc.LostRemote): - client_hbchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_hbchan.recv() print 'CLIENT LOST SERVER :)' client_hbchan.close() server.close() @@ -83,8 +87,11 @@ def test_close_client_hbchan(): gevent.sleep(3) print 'CLOSE CLIENT SOCKET!!!' client_hbchan.close() - with assert_raises(zerorpc.LostRemote): - server_hbchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + server_hbchan.recv() print 'SERVER LOST CLIENT :)' server_hbchan.close() server.close() @@ -111,8 +118,11 @@ def test_heartbeat_can_open_channel_server_close(): gevent.sleep(3) print 'CLOSE SERVER SOCKET!!!' server_hbchan.close() - with assert_raises(zerorpc.LostRemote): - client_hbchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_hbchan.recv() print 'CLIENT LOST SERVER :)' client_hbchan.close() server.close() @@ -140,8 +150,11 @@ def test_heartbeat_can_open_channel_client_close(): print 'CLOSE CLIENT SOCKET!!!' client_hbchan.close() client.close() - with assert_raises(zerorpc.LostRemote): - server_hbchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + server_hbchan.recv() print 'SERVER LOST CLIENT :)' server_hbchan.close() server.close() @@ -209,8 +222,11 @@ def client_do(): assert event.name == 'OK' assert list(event.args) == [x + x * x] client_hbchan.emit('add', (x, x * x)) - with assert_raises(zerorpc.LostRemote): - event = client_hbchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + client_hbchan.recv() client_hbchan.close() client_task = gevent.spawn(client_do) @@ -266,8 +282,11 @@ def server_do(): assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) - with assert_raises(zerorpc.LostRemote): - event = server_hbchan.recv() + if sys.version_info < (2, 7): + assert_raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with assert_raises(zerorpc.LostRemote): + server_hbchan.recv() server_hbchan.close() server_task = gevent.spawn(server_do) @@ -292,12 +311,21 @@ def client_do(): client_channel = client.channel() client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) - with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): - client_hbchan.emit('sleep', (x,)) - event = client_hbchan.recv(timeout=3) - assert event.name == 'OK' - assert list(event.args) == [x] + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in xrange(10): + client_hbchan.emit('sleep', (x,)) + event = client_hbchan.recv(timeout=3) + assert event.name == 'OK' + assert list(event.args) == [x] + assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with assert_raises(zerorpc.TimeoutExpired): + for x in xrange(10): + client_hbchan.emit('sleep', (x,)) + event = client_hbchan.recv(timeout=3) + assert event.name == 'OK' + assert list(event.args) == [x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -307,12 +335,21 @@ def server_do(): server_channel = server.channel(event) server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) - with assert_raises(zerorpc.LostRemote): - for x in xrange(20): - event = server_hbchan.recv() - assert event.name == 'sleep' - gevent.sleep(event.args[0]) - server_hbchan.emit('OK', event.args) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in xrange(20): + event = server_hbchan.recv() + assert event.name == 'sleep' + gevent.sleep(event.args[0]) + server_hbchan.emit('OK', event.args) + assert_raises(zerorpc.LostRemote, _do_with_assert_raises) + else: + with assert_raises(zerorpc.LostRemote): + for x in xrange(20): + event = server_hbchan.recv() + assert event.name == 'sleep' + gevent.sleep(event.args[0]) + server_hbchan.emit('OK', event.args) server_hbchan.close() server_task = gevent.spawn(server_do) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 368eb17..3d55b0f 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -27,7 +27,8 @@ import gevent import gevent.local import random -import md5 +import hashlib +import sys from zerorpc import zmq import zerorpc @@ -94,8 +95,11 @@ def hello(self): return 'world' srv = Srv(heartbeat=1, context=c) - with assert_raises(zmq.ZMQError): - srv.bind('some_service') + if sys.version_info < (2, 7): + assert_raises(zmq.ZMQError, srv.bind, 'some_service') + else: + with assert_raises(zmq.ZMQError): + srv.bind('some_service') cnt = c.register_middleware(Resolver()) assert cnt == 1 @@ -129,7 +133,7 @@ def load_task_context(self, event_header): def get_task_context(self): if self.trace_id is None: # just an ugly code to generate a beautiful little hash. - self._locals.trace_id = '<{0}>'.format(md5.md5( + self._locals.trace_id = '<{0}>'.format(hashlib.md5( str(random.random())[3:] ).hexdigest()[0:6].upper()) print self._identity, 'get_task_context! [make a new one]', self.trace_id diff --git a/tests/test_server.py b/tests/test_server.py index 904d87c..8ac48f0 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -25,6 +25,7 @@ from nose.tools import assert_raises import gevent +import sys from zerorpc import zmq import zerorpc @@ -108,8 +109,11 @@ def add(self, a, b): client = zerorpc.Client(timeout=2) client.connect(endpoint) - with assert_raises(zerorpc.TimeoutExpired): - print client.add(1, 4) + if sys.version_info < (2, 7): + assert_raises(zerorpc.TimeoutExpired, client.add, 1, 4) + else: + with assert_raises(zerorpc.TimeoutExpired): + print client.add(1, 4) client.close() srv.close() @@ -129,8 +133,13 @@ def raise_something(self, a): client = zerorpc.Client(timeout=2) client.connect(endpoint) - with assert_raises(zerorpc.RemoteError): - print client.raise_something(42) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print client.raise_something(42) + assert_raises(zerorpc.RemoteError, _do_with_assert_raises) + else: + with assert_raises(zerorpc.RemoteError): + print client.raise_something(42) assert client.raise_something(range(5)) == 4 client.close() srv.close() @@ -151,8 +160,13 @@ def raise_error(self): client = zerorpc.Client(timeout=2) client.connect(endpoint) - with assert_raises(zerorpc.RemoteError): - print client.raise_error() + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print client.raise_error() + assert_raises(zerorpc.RemoteError, _do_with_assert_raises) + else: + with assert_raises(zerorpc.RemoteError): + print client.raise_error() try: client.raise_error() except zerorpc.RemoteError as e: From 5207316ef44b1451365adfe667e4ac353116f607 Mon Sep 17 00:00:00 2001 From: Mahendra M Date: Wed, 18 Dec 2013 14:59:14 +0530 Subject: [PATCH 022/113] Avoid using dict as default keyword argument --- zerorpc/channel.py | 12 ++++++------ zerorpc/events.py | 11 +++++++---- zerorpc/heartbeat.py | 4 ++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index e304bbc..d6d9648 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -60,13 +60,13 @@ def close(self): if self._channel_dispatcher_task: self._channel_dispatcher_task.kill() - def create_event(self, name, args, xheader={}): + def create_event(self, name, args, xheader=None): return self._events.create_event(name, args, xheader) def emit_event(self, event, identity=None): return self._events.emit_event(event, identity) - def emit(self, name, args, xheader={}): + def emit(self, name, args, xheader=None): return self._events.emit(name, args, xheader) def recv(self): @@ -143,7 +143,7 @@ def close(self): del self._multiplexer._active_channels[self._channel_id] self._channel_id = None - def create_event(self, name, args, xheader={}): + def create_event(self, name, args, xheader=None): event = self._multiplexer.create_event(name, args, xheader) if self._channel_id is None: self._channel_id = event.header['message_id'] @@ -152,7 +152,7 @@ def create_event(self, name, args, xheader={}): event.header['response_to'] = self._channel_id return event - def emit(self, name, args, xheader={}): + def emit(self, name, args, xheader=None): event = self.create_event(name, args, xheader) self._multiplexer.emit_event(event, self._zmqid) @@ -230,7 +230,7 @@ def _recver(self): self.close() return - def create_event(self, name, args, xheader={}): + def create_event(self, name, args, xheader=None): return self._channel.create_event(name, args, xheader) def emit_event(self, event, block=True, timeout=None): @@ -247,7 +247,7 @@ def emit_event(self, event, block=True, timeout=None): raise return True - def emit(self, name, args, xheader={}, block=True, timeout=None): + def emit(self, name, args, xheader=None, block=True, timeout=None): event = self.create_event(name, args, xheader) return self.emit_event(event, block, timeout) diff --git a/zerorpc/events.py b/zerorpc/events.py index e4d79ee..93a0eb1 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -226,7 +226,8 @@ def bind(self, endpoint, resolve=True): r.append(self._socket.bind(endpoint_)) return r - def create_event(self, name, args, xheader={}): + def create_event(self, name, args, xheader=None): + xheader = {} if xheader is None else xheader event = Event(name, args, context=self._context) for k, v in xheader.items(): if k == 'zmqid': @@ -244,7 +245,8 @@ def emit_event(self, event, identity=None): parts = (event.pack(),) self._send(parts) - def emit(self, name, args, xheader={}): + def emit(self, name, args, xheader=None): + xheader = {} if xheader is None else xheader event = self.create_event(name, args, xheader) identity = xheader.get('zmqid', None) return self.emit_event(event, identity) @@ -282,7 +284,8 @@ def close(self): def recv_is_available(self): return self._channel.recv_is_available - def create_event(self, name, args, xheader={}): + def create_event(self, name, args, xheader=None): + xheader = {} if xheader is None else xheader event = Event(name, args, self._channel.context) event.header.update(xheader) return event @@ -292,7 +295,7 @@ def emit_event(self, event, identity=None): wrapper_event = self._channel.create_event('w', event_payload) self._channel.emit_event(wrapper_event) - def emit(self, name, args, xheader={}): + def emit(self, name, args, xheader=None): wrapper_event = self.create_event(name, args, xheader) self.emit_event(wrapper_event) diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index 92421af..987ce13 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -100,7 +100,7 @@ def _lost_remote_exception(self): return LostRemote('Lost remote after {0}s heartbeat'.format( self._heartbeat_freq * 2)) - def create_event(self, name, args, xheader={}): + def create_event(self, name, args, xheader=None): if self._compat_v2 and name == '_zpc_more': name = '_zpc_hb' return self._channel.create_event(name, args, xheader) @@ -110,7 +110,7 @@ def emit_event(self, event): raise self._lost_remote_exception() self._channel.emit_event(event) - def emit(self, name, args, xheader={}): + def emit(self, name, args, xheader=None): event = self.create_event(name, args, xheader) self.emit_event(event) From 05a17bb4891f3ee9b36b06b1a5d112f303d1db7f Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 30 Dec 2013 11:35:27 +0200 Subject: [PATCH 023/113] Using new style classes in code example. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b44fe01..21eba59 100644 --- a/README.rst +++ b/README.rst @@ -156,7 +156,7 @@ python API. Below are a few examples. Here's how to expose an object of your choice as a zeroservice:: - class Cooler: + class Cooler(object): """ Various convenience methods to make things cooler. """ def add_man(self, sentence): From f1e8140c44fb1ce81df800ce20b50fbcb0c1dd38 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 30 Dec 2013 22:48:25 +0200 Subject: [PATCH 024/113] Travis will now test against different versions of pyzmq. --- .travis.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2a1a60c..5eb33c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,22 @@ language: python python: - 2.7 +env: + matrix: + - PYZMQ='pyzmq==14.0.1' + - PYZMQ='pyzmq==14.0.0' + - PYZMQ='pyzmq==13.1.0' + - PYZMQ='pyzmq==13.0.2' + - PYZMQ='pyzmq==13.0.0' +matrix: + fast_finish: true + allow_failures: + - env: PYZMQ='pyzmq==14.0.0' + - env: PYZMQ='pyzmq==14.0.1' script: nosetests before_install: - sudo apt-get update - sudo apt-get install python-dev libevent-dev install: - - pip install nose --use-mirrors - - pip install . --use-mirrors + - "pip install nose gevent msgpack-python $PYZMQ" + - pip install . --no-deps From 8b219a10d65a15383a556ca2cde87f18c8571b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Fri, 7 Mar 2014 12:16:40 -0800 Subject: [PATCH 025/113] Bump up gevent requirement to >=1.0. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4e566a9..0c0b5e3 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ requirements = [ - 'gevent', + 'gevent>=1.0', 'msgpack-python', 'pyzmq==13.1.0' ] From f435520572cd8202821a978607f79fd6906dfe05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Fri, 7 Mar 2014 12:44:02 -0800 Subject: [PATCH 026/113] Cosmetic adjustments to make flake8 happy. --- zerorpc/__init__.py | 2 ++ zerorpc/channel.py | 25 +++++++++++-------------- zerorpc/cli.py | 15 +++++++++------ zerorpc/context.py | 16 ++++++++-------- zerorpc/core.py | 15 ++++++++------- zerorpc/decorators.py | 2 +- zerorpc/events.py | 8 ++++---- zerorpc/gevent_zmq.py | 18 +++++++++--------- zerorpc/heartbeat.py | 2 +- 9 files changed, 53 insertions(+), 50 deletions(-) diff --git a/zerorpc/__init__.py b/zerorpc/__init__.py index 4e1b040..505ba3c 100644 --- a/zerorpc/__init__.py +++ b/zerorpc/__init__.py @@ -22,6 +22,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# Tell flake8 to ignore this file (otherwise it will complain about import *) +# flake8: noqa from .version import * from .exceptions import * from .context import * diff --git a/zerorpc/channel.py b/zerorpc/channel.py index d6d9648..5da8991 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -22,8 +22,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import sys - import gevent.pool import gevent.queue import gevent.event @@ -37,7 +35,6 @@ logger = getLogger(__name__) - class ChannelMultiplexer(object): def __init__(self, events, ignore_broadcast=False): self._events = events @@ -81,9 +78,9 @@ def _channel_dispatcher(self): try: event = self._events.recv() except Exception as e: - logger.error( \ - 'zerorpc.ChannelMultiplexer, ' + \ - 'ignoring error on recv: {0}'.format(e)) + logger.error( + 'zerorpc.ChannelMultiplexer, ' + 'ignoring error on recv: {0}'.format(e)) continue channel_id = event.header.get('response_to', None) @@ -96,10 +93,10 @@ def _channel_dispatcher(self): queue = self._broadcast_queue if queue is None: - logger.error( \ - 'zerorpc.ChannelMultiplexer, ' + \ - 'unable to route event: ' + \ - event.__str__(ignore_args=True)) + logger.error( + 'zerorpc.ChannelMultiplexer, ' + 'unable to route event: {0}' + .format(event.__str__(ignore_args=True))) else: queue.put(event) @@ -215,14 +212,14 @@ def _recver(self): try: self._remote_queue_open_slots += int(event.args[0]) except Exception as e: - logger.error( \ - 'gevent_zerorpc.BufferedChannel._recver, ' + \ - 'exception: ' + repr(e)) + logger.error( + 'gevent_zerorpc.BufferedChannel._recver, ' + 'exception: ' + repr(e)) if self._remote_queue_open_slots > 0: self._remote_can_recv.set() elif self._input_queue.qsize() == self._input_queue_size: raise RuntimeError( - 'BufferedChannel, queue overflow on event:', event) + 'BufferedChannel, queue overflow on event:', event) else: self._input_queue.put(event) if self._on_close_if is not None and self._on_close_if(event): diff --git a/zerorpc/cli.py b/zerorpc/cli.py index d32ed2f..b7e6135 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -35,8 +35,8 @@ parser = argparse.ArgumentParser( - description='Make a zerorpc call to a remote service.' - ) + description='Make a zerorpc call to a remote service.' +) client_or_server = parser.add_mutually_exclusive_group() client_or_server.add_argument('--client', action='store_true', default=True, @@ -139,11 +139,12 @@ def remote_detailled_methods(): remote_detailled_methods())) r = [(name + (inspect.formatargspec(*argspec) - if argspec else '(...)'), doc) - for name, argspec, doc in remote_detailled_methods()] + if argspec else '(...)'), doc) + for name, argspec, doc in remote_detailled_methods()] longest_name_len = max(len(name) for name, doc in r) return (longest_name_len, r) + # handle the 'python formatted' _zerorpc_inspect, that return the output of # "getargspec" from the python lib "inspect". def zerorpc_inspect_python_argspecs(remote_methods, filter_method, long_doc, include_argspec): @@ -157,10 +158,11 @@ def format_method(name, argspec, doc): doc = doc.splitlines()[0] return (name, doc) r = [format_method(*methods_info) for methods_info in remote_methods if - filter_method is None or methods_info[0] == filter_method] + filter_method is None or methods_info[0] == filter_method] longest_name_len = max(len(name) for name, doc in r) return (longest_name_len, r) + def zerorpc_inspect_generic(remote_methods, filter_method, long_doc, include_argspec): def format_method(name, args, doc): if include_argspec: @@ -183,6 +185,7 @@ def format_arg(arg): longest_name_len = max(len(name) for name, doc in methods) return (longest_name_len, methods) + def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): try: remote_methods = client._zerorpc_inspect()['methods'] @@ -201,6 +204,7 @@ def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): return zerorpc_inspect_generic(remote_methods, method, long_doc, include_argspec) + def run_client(args): client = zerorpc.Client(timeout=args.timeout, heartbeat=args.heartbeat, passive_heartbeat=not args.active_hb) @@ -269,4 +273,3 @@ def main(): return -1 return run_server(args) - diff --git a/zerorpc/context.py b/zerorpc/context.py index f489509..879e6c4 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -100,8 +100,8 @@ def get_instance(): def _reset_msgid(self): self._msg_id_base = str(uuid.uuid4())[8:] - self._msg_id_counter = random.randrange(0, 2**32) - self._msg_id_counter_stop = random.randrange(self._msg_id_counter, 2**32) + self._msg_id_counter = random.randrange(0, 2 ** 32) + self._msg_id_counter_stop = random.randrange(self._msg_id_counter, 2 ** 32) def new_msgid(self): if self._msg_id_counter >= self._msg_id_counter_stop: @@ -125,9 +125,9 @@ def register_middleware(self, middleware_instance): registered_count += 1 return registered_count - + # # client/server - + # def hook_resolve_endpoint(self, endpoint): for functor in self._hooks['resolve_endpoint']: endpoint = functor(endpoint) @@ -143,9 +143,9 @@ def hook_get_task_context(self): event_header.update(functor()) return event_header - + # # Server-side hooks - + # def hook_server_before_exec(self, request_event): """Called when a method is about to be executed on the server.""" @@ -176,9 +176,9 @@ def hook_server_inspect_exception(self, request_event, reply_event, exc_infos): for functor in self._hooks['server_inspect_exception']: functor(request_event, reply_event, task_context, exc_infos) - + # # Client-side hooks - + # def hook_client_handle_remote_error(self, event): exception = None for functor in self._hooks['client_handle_remote_error']: diff --git a/zerorpc/core.py b/zerorpc/core.py index 2be8067..f30d076 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -71,13 +71,13 @@ def _filter_methods(cls, self, methods): if hasattr(methods, '__getitem__'): return methods server_methods = set(getattr(self, k) for k in dir(cls) if not - k.startswith('_')) + k.startswith('_')) return dict((k, getattr(methods, k)) - for k in dir(methods) - if callable(getattr(methods, k)) - and not k.startswith('_') - and getattr(methods, k) not in server_methods - ) + for k in dir(methods) + if (callable(getattr(methods, k)) + and not k.startswith('_') + and getattr(methods, k) not in server_methods + )) @staticmethod def _extract_name(methods): @@ -111,7 +111,7 @@ def _inject_builtins(self): self._methods['_zerorpc_name'] = lambda: self._name self._methods['_zerorpc_ping'] = lambda: ['pong', self._name] self._methods['_zerorpc_help'] = lambda m: \ - self._methods[m]._zerorpc_doc() + self._methods[m]._zerorpc_doc() self._methods['_zerorpc_args'] = \ lambda m: self._methods[m]._zerorpc_args() self._methods['_zerorpc_inspect'] = self._zerorpc_inspect @@ -406,6 +406,7 @@ def fork_task_context(functor, context=None): ''' context = context or Context.get_instance() header = context.hook_get_task_context() + def wrapped(*args, **kargs): context.hook_load_task_context(header) return functor(*args, **kargs) diff --git a/zerorpc/decorators.py b/zerorpc/decorators.py index 3d20c71..f662e7e 100644 --- a/zerorpc/decorators.py +++ b/zerorpc/decorators.py @@ -24,7 +24,7 @@ import inspect -from .patterns import * +from .patterns import * # noqa class DecoratorBase(object): diff --git a/zerorpc/events.py b/zerorpc/events.py index 93a0eb1..c358951 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -107,16 +107,16 @@ def __call__(self): class Event(object): - __slots__ = [ '_name', '_args', '_header' ] + __slots__ = ['_name', '_args', '_header'] def __init__(self, name, args, context, header=None): self._name = name self._args = args if header is None: self._header = { - 'message_id': context.new_msgid(), - 'v': 3 - } + 'message_id': context.new_msgid(), + 'v': 3 + } else: self._header = header diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index edda633..a5cd67e 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -25,19 +25,19 @@ # Based on https://github.com/traviscline/gevent-zeromq/ # We want to act like zmq -from zmq import * +from zmq import * # noqa # A way to access original zmq import zmq as _zmq import gevent.event import gevent.core -import sys import errno from logging import getLogger logger = getLogger(__name__) + class Context(_zmq.Context): def socket(self, socket_type): @@ -59,13 +59,13 @@ def __init__(self, context, socket_type): try: # gevent>=1.0 self.__dict__["_state_event"] = gevent.hub.get_hub().loop.io( - on_state_changed_fd, gevent.core.READ) + on_state_changed_fd, gevent.core.READ) self._state_event.start(self._on_state_changed) except AttributeError: # gevent<1.0 self.__dict__["_state_event"] = \ - gevent.core.read_event(on_state_changed_fd, - self._on_state_changed, persist=True) + gevent.core.read_event(on_state_changed_fd, + self._on_state_changed, persist=True) def _on_state_changed(self, event=None, _evtype=None): if self.closed: @@ -137,8 +137,8 @@ def send(self, data, flags=0, copy=True, track=False): while not self._writable.wait(timeout=1): try: if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: - logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ - "catching up after missing event (SEND) /!\\") + logger.error("/!\\ gevent_zeromq BUG /!\\ " + "catching up after missing event (SEND) /!\\") break except ZMQError as e: if e.errno not in (_zmq.EAGAIN, errno.EINTR): @@ -177,8 +177,8 @@ def recv(self, flags=0, copy=True, track=False): while not self._readable.wait(timeout=1): try: if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: - logger.error("/!\\ gevent_zeromq BUG /!\\ " + \ - "catching up after missing event (RECV) /!\\") + logger.error("/!\\ gevent_zeromq BUG /!\\ " + "catching up after missing event (RECV) /!\\") break except ZMQError as e: if e.errno not in (_zmq.EAGAIN, errno.EINTR): diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index 987ce13..c4e7c4b 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -30,7 +30,7 @@ import gevent.local import gevent.lock -from .exceptions import * +from .exceptions import * # noqa class HeartBeatOnChannel(object): From 362886f24ae473f5455a0343b8d5dbdc87b8ab32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Fri, 7 Mar 2014 15:51:47 -0800 Subject: [PATCH 027/113] Add flake8 to tox. --- tox.ini | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a9a789f..6f68c02 100644 --- a/tox.ini +++ b/tox.ini @@ -2,5 +2,9 @@ envlist = py26,py27,py31,py32 [testenv] -deps=nose -commands=nosetests +deps = + flake8 + nose +commands = + flake8 --ignore=E501,E128 zerorpc bin + nosetests From ffe29738f070564142ed4cb5f073b6ee0625782b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Fri, 7 Mar 2014 15:51:56 -0800 Subject: [PATCH 028/113] Add flake8 to Travis CI. --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5eb33c0..e3546e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,13 @@ matrix: allow_failures: - env: PYZMQ='pyzmq==14.0.0' - env: PYZMQ='pyzmq==14.0.1' -script: nosetests +script: + - flake8 --ignore=E501,E128 zerorpc bin + - nosetests before_install: - sudo apt-get update - sudo apt-get install python-dev libevent-dev install: + - pip install flake8 - "pip install nose gevent msgpack-python $PYZMQ" - pip install . --no-deps From 1f4f8de537b3b21ea91efd9b53ccace8fa971d21 Mon Sep 17 00:00:00 2001 From: Lorenzo Villani Date: Sat, 19 Apr 2014 19:52:50 +0200 Subject: [PATCH 029/113] Replace execfile with something that works under both Python 2 and 3 Execfile doesn't exist in Python 3, this commit replaces the use of execfile with an equivalent construct known to work under both versions of Python. The change is actually the result of running 2to3 over setup.py, I just added a note to explain its presence. --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c0b5e3..13d4149 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -execfile('zerorpc/version.py') +# execfile() doesn't exist in Python 3, this way we are compatible with both. +exec(compile(open('zerorpc/version.py').read(), 'zerorpc/version.py', 'exec')) + import sys From c08e12a5d83ee9b147a2474c407f2d4504ff4c40 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 6 May 2014 06:12:52 -0400 Subject: [PATCH 030/113] Fix name of reply header After looking at the code (while working on a haskell implementation), I noticed that it uses 'response_to' rather than 'reply_to' as the response message header. --- doc/protocol.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/protocol.md b/doc/protocol.md index 6bd2ad7..2dc3bc0 100644 --- a/doc/protocol.md +++ b/doc/protocol.md @@ -103,12 +103,12 @@ This document talks only about the version 3 of the protocol. - Each new event opens a new channel implicitly. - The id of the new event will represent the channel id for the connection. - - Each consecutive event on a channel will have the header field "reply_to" + - Each consecutive event on a channel will have the header field "response_to" set to the channel id: { "message_id": "6ce9503a-bfb8-486a-ac79-e2ed225ace79", - "reply_to": "6636fb60-2bca-4ecb-8cb4-bbaaf174e9e6" + "response_to": "6636fb60-2bca-4ecb-8cb4-bbaaf174e9e6" } #### Heartbeat From df1a5bc60ebf4575569eb5439d08e53649954e0a Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 8 Jun 2014 10:55:49 +0300 Subject: [PATCH 031/113] Added the latest versions of pyzmq to the build process. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index e3546e4..4e232b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ python: - 2.7 env: matrix: + - PYZMQ='pyzmq==14.1.1' + - PYZMQ='pyzmq==14.1.0' - PYZMQ='pyzmq==14.0.1' - PYZMQ='pyzmq==14.0.0' - PYZMQ='pyzmq==13.1.0' @@ -13,6 +15,8 @@ matrix: allow_failures: - env: PYZMQ='pyzmq==14.0.0' - env: PYZMQ='pyzmq==14.0.1' + - env: PYZMQ='pyzmq==14.1.0' + - env: PYZMQ='pyzmq==14.1.1' script: - flake8 --ignore=E501,E128 zerorpc bin - nosetests From 273853eba9cd73092e16fa74cd5519dae5b31de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 00:04:52 -0700 Subject: [PATCH 032/113] Faster unit tests! One can set the env variable ZPC_TEST_TIME_FACTOR to something smaller than 1.0 in order to speedup testing. At some really small values, unit tests about timeouts are likely to fail! Example: $ ZPC_TEST_TIME_FACTOR=0.01 nosetests --- tests/test_buffered_channel.py | 78 ++++++++++------------ tests/test_client_async.py | 6 +- tests/test_client_heartbeat.py | 50 +++++++------- tests/test_heartbeat.py | 50 +++++++------- tests/test_middleware.py | 12 ++-- tests/test_middleware_before_after_exec.py | 12 ++-- tests/test_middleware_client.py | 10 +-- tests/test_reqstream.py | 8 +-- tests/test_server.py | 10 +-- tests/testutils.py | 6 ++ 10 files changed, 120 insertions(+), 122 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index f55993c..93c7db2 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -29,7 +29,7 @@ from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_bufchan(): @@ -43,17 +43,17 @@ def test_close_server_bufchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE SERVER SOCKET!!!' server_bufchan.close() if sys.version_info < (2, 7): @@ -78,17 +78,17 @@ def test_close_client_bufchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE CLIENT SOCKET!!!' client_bufchan.close() if sys.version_info < (2, 7): @@ -113,15 +113,15 @@ def test_heartbeat_can_open_channel_server_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE SERVER SOCKET!!!' server_bufchan.close() if sys.version_info < (2, 7): @@ -146,15 +146,15 @@ def test_heartbeat_can_open_channel_client_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE CLIENT SOCKET!!!' client_bufchan.close() client.close() @@ -179,12 +179,12 @@ def test_do_some_req_rep(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) def client_do(): @@ -225,7 +225,7 @@ def test_do_some_req_rep_lost_server(): def client_do(): print 'running' client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) for x in xrange(10): client_bufchan.emit('add', (x, x * x)) @@ -246,7 +246,7 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) for x in xrange(10): event = server_bufchan.recv() @@ -273,7 +273,7 @@ def test_do_some_req_rep_lost_client(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) for x in xrange(10): @@ -289,7 +289,7 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) for x in xrange(10): @@ -323,14 +323,14 @@ def test_do_some_req_rep_client_timeout(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(10): client_bufchan.emit('sleep', (x,)) - event = client_bufchan.recv(timeout=3) + event = client_bufchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' assert list(event.args) == [x] assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) @@ -338,7 +338,7 @@ def _do_with_assert_raises(): with assert_raises(zerorpc.TimeoutExpired): for x in xrange(10): client_bufchan.emit('sleep', (x,)) - event = client_bufchan.recv(timeout=3) + event = client_bufchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' assert list(event.args) == [x] client_bufchan.close() @@ -349,7 +349,7 @@ def _do_with_assert_raises(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) if sys.version_info < (2, 7): @@ -357,7 +357,7 @@ def _do_with_assert_raises(): for x in xrange(20): event = server_bufchan.recv() assert event.name == 'sleep' - gevent.sleep(event.args[0]) + gevent.sleep(TIME_FACTOR * event.args[0]) server_bufchan.emit('OK', event.args) assert_raises(zerorpc.LostRemote, _do_with_assert_raises) else: @@ -365,7 +365,7 @@ def _do_with_assert_raises(): for x in xrange(20): event = server_bufchan.recv() assert event.name == 'sleep' - gevent.sleep(event.args[0]) + gevent.sleep(TIME_FACTOR * event.args[0]) server_bufchan.emit('OK', event.args) server_bufchan.close() @@ -377,10 +377,6 @@ def _do_with_assert_raises(): server.close() -class CongestionError(Exception): - pass - - def test_congestion_control_server_pushing(): endpoint = random_ipc_endpoint() server_events = zerorpc.Events(zmq.ROUTER) @@ -392,12 +388,12 @@ def test_congestion_control_server_pushing(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) def client_do(): @@ -413,26 +409,22 @@ def server_do(): if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(200): - if not server_bufchan.emit('coucou', x, block=False): - raise CongestionError() # will fail when x == 1 - assert_raises(CongestionError, _do_with_assert_raises) + server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 1 + assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: - with assert_raises(CongestionError): + with assert_raises(zerorpc.TimeoutExpired): for x in xrange(200): - if not server_bufchan.emit('coucou', x, block=False): - raise CongestionError() # will fail when x == 1 + server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 1 server_bufchan.emit('coucou', 1) # block until receiver is ready if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(2, 200): - if not server_bufchan.emit('coucou', x, block=False): - raise CongestionError() # will fail when x == 100 - assert_raises(CongestionError, _do_with_assert_raises) + server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 100 + assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: - with assert_raises(CongestionError): + with assert_raises(zerorpc.TimeoutExpired): for x in xrange(2, 200): - if not server_bufchan.emit('coucou', x, block=False): - raise CongestionError() # will fail when x == 100 + server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 100 for x in xrange(101, 200): server_bufchan.emit('coucou', x) # block until receiver is ready diff --git a/tests/test_client_async.py b/tests/test_client_async.py index 76e0abe..a9b1fb6 100644 --- a/tests/test_client_async.py +++ b/tests/test_client_async.py @@ -29,7 +29,7 @@ from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_client_server_client_timeout_with_async(): @@ -41,14 +41,14 @@ def lolita(self): return 42 def add(self, a, b): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) return a + b srv = MySrv() srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) async_result = client.add(1, 4, async=True) diff --git a/tests/test_client_heartbeat.py b/tests/test_client_heartbeat.py index e9de869..d884aee 100644 --- a/tests/test_client_heartbeat.py +++ b/tests/test_client_heartbeat.py @@ -26,7 +26,7 @@ import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_client_server_hearbeat(): @@ -38,13 +38,13 @@ def lolita(self): return 42 def slow(self): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 1) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=1) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1) client.connect(endpoint) assert client.lolita() == 42 @@ -57,15 +57,15 @@ def test_client_server_activate_heartbeat(): class MySrv(zerorpc.Server): def lolita(self): - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) return 42 - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 1) srv.bind(endpoint) gevent.spawn(srv.run) - gevent.sleep(0) + gevent.sleep(TIME_FACTOR * 0) - client = zerorpc.Client(heartbeat=1) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1) client.connect(endpoint) assert client.lolita() == 42 @@ -81,14 +81,14 @@ def lolita(self): return 42 def slow(self): - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) return 2 - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 1) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=1, passive_heartbeat=True) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1, passive_heartbeat=True) client.connect(endpoint) assert client.slow() == 2 @@ -104,16 +104,16 @@ class MySrv(zerorpc.Server): def iter(self): return xrange(42) - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): assert list(client1.iter()) == list(xrange(42)) print 'sleep 3s' - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -126,18 +126,18 @@ class MySrv(zerorpc.Server): def lolita(self): return 42 - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) - client2 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) - client3 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) + client2 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) + client3 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) assert client1.lolita() == 42 assert client2.lolita() == 42 - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) assert client3.lolita() == 42 @@ -150,18 +150,18 @@ class MySrv(zerorpc.Server): def iter(self): return [] - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): print 'grab iter' i = client1.iter() print 'sleep 3s' - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -175,11 +175,11 @@ class MySrv(zerorpc.Server): def iter(self): return xrange(500) - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): print 'grab iter' @@ -189,6 +189,6 @@ def test_client(): assert list(next(i) for x in xrange(142)) == list(xrange(142)) print 'sleep 3s' - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index a4489e1..f6234f7 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -29,7 +29,7 @@ from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_hbchan(): @@ -43,15 +43,15 @@ def test_close_server_hbchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_hbchan.recv() - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE SERVER SOCKET!!!' server_hbchan.close() if sys.version_info < (2, 7): @@ -76,15 +76,15 @@ def test_close_client_hbchan(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_hbchan.recv() - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE CLIENT SOCKET!!!' client_hbchan.close() if sys.version_info < (2, 7): @@ -109,13 +109,13 @@ def test_heartbeat_can_open_channel_server_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE SERVER SOCKET!!!' server_hbchan.close() if sys.version_info < (2, 7): @@ -140,13 +140,13 @@ def test_heartbeat_can_open_channel_client_close(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) print 'CLOSE CLIENT SOCKET!!!' client_hbchan.close() client.close() @@ -171,11 +171,11 @@ def test_do_some_req_rep(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) def client_do(): for x in xrange(20): @@ -215,7 +215,7 @@ def test_do_some_req_rep_lost_server(): def client_do(): print 'running' client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) for x in xrange(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() @@ -234,7 +234,7 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) for x in xrange(10): event = server_hbchan.recv() assert event.name == 'add' @@ -261,7 +261,7 @@ def test_do_some_req_rep_lost_client(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) for x in xrange(10): client_hbchan.emit('add', (x, x * x)) @@ -275,7 +275,7 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) for x in xrange(10): event = server_hbchan.recv() @@ -309,13 +309,13 @@ def test_do_some_req_rep_client_timeout(): def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(10): client_hbchan.emit('sleep', (x,)) - event = client_hbchan.recv(timeout=3) + event = client_hbchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' assert list(event.args) == [x] assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) @@ -323,7 +323,7 @@ def _do_with_assert_raises(): with assert_raises(zerorpc.TimeoutExpired): for x in xrange(10): client_hbchan.emit('sleep', (x,)) - event = client_hbchan.recv(timeout=3) + event = client_hbchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' assert list(event.args) == [x] client_hbchan.close() @@ -333,14 +333,14 @@ def _do_with_assert_raises(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(20): event = server_hbchan.recv() assert event.name == 'sleep' - gevent.sleep(event.args[0]) + gevent.sleep(TIME_FACTOR * event.args[0]) server_hbchan.emit('OK', event.args) assert_raises(zerorpc.LostRemote, _do_with_assert_raises) else: @@ -348,7 +348,7 @@ def _do_with_assert_raises(): for x in xrange(20): event = server_hbchan.recv() assert event.name == 'sleep' - gevent.sleep(event.args[0]) + gevent.sleep(TIME_FACTOR * event.args[0]) server_hbchan.emit('OK', event.args) server_hbchan.close() diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3d55b0f..c1bb073 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -32,7 +32,7 @@ from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_resolve_endpoint(): @@ -94,7 +94,7 @@ def hello(self): print 'heee' return 'world' - srv = Srv(heartbeat=1, context=c) + srv = Srv(heartbeat=TIME_FACTOR * 1, context=c) if sys.version_info < (2, 7): assert_raises(zmq.ZMQError, srv.bind, 'some_service') else: @@ -106,7 +106,7 @@ def hello(self): srv.bind('some_service') gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=1, context=c) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1, context=c) client.connect('some_service') assert client.hello() == 'world' @@ -373,9 +373,9 @@ def echo(self, msg): trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). - for attempt in xrange(0, 10): + for attempt in xrange(0, 100): c.echo('pub...') - if trigger.wait(0.2): + if trigger.wait(TIME_FACTOR * 0.2): break subscriber.stop() @@ -459,7 +459,7 @@ def test_server_inspect_exception_middleware_puller(): barrier.clear() client.echo('This is a test which should call the InspectExceptionMiddleware') - barrier.wait(timeout=2) + barrier.wait(timeout=TIME_FACTOR * 2) client.close() server.close() diff --git a/tests/test_middleware_before_after_exec.py b/tests/test_middleware_before_after_exec.py index 23c2e92..e33abfd 100644 --- a/tests/test_middleware_before_after_exec.py +++ b/tests/test_middleware_before_after_exec.py @@ -25,7 +25,7 @@ import gevent import zerorpc -from testutils import random_ipc_endpoint +from testutils import random_ipc_endpoint, TIME_FACTOR class EchoModule(object): @@ -91,7 +91,7 @@ def test_hook_server_before_exec_puller(): # Test without a middleware test_client.echo("test") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test" trigger.clear() @@ -100,7 +100,7 @@ def test_hook_server_before_exec_puller(): zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False test_client.echo("test with a middleware") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test with a middleware" assert test_middleware.called == True @@ -183,7 +183,7 @@ def test_hook_server_after_exec_puller(): # Test without a middleware test_client.echo("test") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test" trigger.clear() @@ -192,7 +192,7 @@ def test_hook_server_after_exec_puller(): zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False test_client.echo("test with a middleware") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test with a middleware" assert test_middleware.called == True assert test_middleware.request_event_name == 'echo' @@ -288,7 +288,7 @@ def test_hook_server_after_exec_on_error_puller(): assert test_middleware.called == False try: test_client.echo("test with a middleware") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) except zerorpc.RemoteError: pass assert echo_module.last_msg == "Raise" diff --git a/tests/test_middleware_client.py b/tests/test_middleware_client.py index 0315100..828ee85 100644 --- a/tests/test_middleware_client.py +++ b/tests/test_middleware_client.py @@ -25,7 +25,7 @@ import gevent import zerorpc -from testutils import random_ipc_endpoint +from testutils import random_ipc_endpoint, TIME_FACTOR class EchoModule(object): @@ -59,7 +59,7 @@ def echoes_crash(self, msg): def timeout(self, msg): self.last_msg = "timeout: " + msg - gevent.sleep(2) + gevent.sleep(TIME_FACTOR * 2) def test_hook_client_before_request(): @@ -175,7 +175,7 @@ def client_after_request(self, req_event, rep_event, exception): test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) - test_client = zerorpc.Client(timeout=1, context=zero_ctx) + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False @@ -211,7 +211,7 @@ def test_hook_client_after_request_remote_error(): test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) - test_client = zerorpc.Client(timeout=1, context=zero_ctx) + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False @@ -234,7 +234,7 @@ def test_hook_client_after_request_remote_error_stream(): test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) - test_client = zerorpc.Client(timeout=1, context=zero_ctx) + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index ee30423..a06c4f3 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -26,7 +26,7 @@ import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_rcp_streaming(): @@ -42,11 +42,11 @@ def range(self, max): def xrange(self, max): return xrange(max) - srv = MySrv(heartbeat=2) + srv = MySrv(heartbeat=TIME_FACTOR * 2) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=2) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 2) client.connect(endpoint) r = client.range(10) @@ -56,7 +56,7 @@ def xrange(self, max): assert getattr(r, 'next', None) is not None l = [] print 'wait 4s for fun' - gevent.sleep(4) + gevent.sleep(TIME_FACTOR * 4) for x in r: l.append(x) assert l == range(10) diff --git a/tests/test_server.py b/tests/test_server.py index 8ac48f0..89c5476 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -29,7 +29,7 @@ from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_server_manual(): @@ -99,14 +99,14 @@ def lolita(self): return 42 def add(self, a, b): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) return a + b srv = MySrv() srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) if sys.version_info < (2, 7): @@ -130,7 +130,7 @@ def raise_something(self, a): srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) if sys.version_info < (2, 7): @@ -157,7 +157,7 @@ def raise_error(self): srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) if sys.version_info < (2, 7): diff --git a/tests/testutils.py b/tests/testutils.py index 175967c..d592e05 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -52,3 +52,9 @@ def wrap(): raise nose.exc.SkipTest(reason) return wrap return _skip + +try: + TIME_FACTOR = float(os.environ.get('ZPC_TEST_TIME_FACTOR')) + print 'ZPC_TEST_TIME_FACTOR:', TIME_FACTOR +except TypeError: + TIME_FACTOR = 1.0 From d88b96c3c796c800e88a67751d5c6849ccf63bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 00:16:32 -0700 Subject: [PATCH 033/113] Run tests on travis 10 times faster. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4e232b4..d162685 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ matrix: - env: PYZMQ='pyzmq==14.1.1' script: - flake8 --ignore=E501,E128 zerorpc bin - - nosetests + - ZPC_TEST_TIME_FACTOR=0.1 nosetests before_install: - sudo apt-get update - sudo apt-get install python-dev libevent-dev From 664a315be97868940471790585a72d7ad521ed5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 01:17:27 -0700 Subject: [PATCH 034/113] Initialize the logger with basic config in the cli --- zerorpc/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zerorpc/cli.py b/zerorpc/cli.py index b7e6135..b542ad7 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -29,6 +29,7 @@ import sys import inspect import os +import logging from pprint import pprint import zerorpc @@ -253,6 +254,7 @@ def run_client(args): def main(): + logging.basicConfig() args = parser.parse_args() if args.bind or args.connect: From c4976051c93d30fb0b683886f3ae7a66406c40b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 01:06:55 -0700 Subject: [PATCH 035/113] Remove useless wrapped event --- tests/test_wrapped_events.py | 202 ----------------------------------- zerorpc/events.py | 37 ------- 2 files changed, 239 deletions(-) delete mode 100644 tests/test_wrapped_events.py diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py deleted file mode 100644 index 43c42fe..0000000 --- a/tests/test_wrapped_events.py +++ /dev/null @@ -1,202 +0,0 @@ -# -*- coding: utf-8 -*- -# Open Source Initiative OSI - The MIT License (MIT):Licensing -# -# The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import random -import gevent - -from zerorpc import zmq -import zerorpc -from testutils import teardown, random_ipc_endpoint - - -def test_sub_events(): - endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.ROUTER) - server_events.bind(endpoint) - server = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.DEALER) - client_events.connect(endpoint) - client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - - client_channel = client.channel() - client_channel_events = zerorpc.WrappedEvents(client_channel) - client_channel_events.emit('coucou', 42) - - event = server.recv() - print event - assert isinstance(event.args, (list, tuple)) - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - server_channel_channel = zerorpc.ChannelMultiplexer(server_channel_events) - event = server_channel_channel.recv() - print event - assert event.name == 'coucou' - assert event.args == 42 - - server_events.close() - client_events.close() - - -def test_multiple_sub_events(): - endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.ROUTER) - server_events.bind(endpoint) - server = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.DEALER) - client_events.connect(endpoint) - client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - - client_channel1 = client.channel() - client_channel_events1 = zerorpc.WrappedEvents(client_channel1) - client_channel2 = zerorpc.BufferedChannel(client.channel()) - client_channel_events2 = zerorpc.WrappedEvents(client_channel2) - - def emitstuff(): - client_channel_events1.emit('coucou1', 43) - client_channel_events2.emit('coucou2', 44) - client_channel_events2.emit('another', 42) - gevent.spawn(emitstuff) - - event = server.recv() - print event - assert isinstance(event.args, (list, tuple)) - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - event = server_channel_events.recv() - print 'ch1:', event - assert event.name == 'coucou1' - assert event.args == 43 - - event = server.recv() - print event - assert isinstance(event.args, (list, tuple)) - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - - server_channel_events = zerorpc.BufferedChannel(server_channel) - server_channel_events = zerorpc.WrappedEvents(server_channel_events) - event = server_channel_events.recv() - print 'ch2:', event - assert event.name == 'coucou2' - assert event.args == 44 - - event = server_channel_events.recv() - print 'ch2:', event - assert event.name == 'another' - assert event.args == 42 - - server_events.close() - client_events.close() - - -def test_recursive_multiplexer(): - endpoint = random_ipc_endpoint() - - server_events = zerorpc.Events(zmq.ROUTER) - server_events.bind(endpoint) - servermux = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.DEALER) - client_events.connect(endpoint) - clientmux = zerorpc.ChannelMultiplexer(client_events, - ignore_broadcast=True) - - def ping_pong(climux, srvmux): - cli_chan = climux.channel() - someid = random.randint(0, 1000000) - print 'ping...' - cli_chan.emit('ping', someid) - print 'srv_chan got:' - event = srvmux.recv() - srv_chan = srvmux.channel(event) - print event - assert event.name == 'ping' - assert event.args == someid - print 'pong...' - srv_chan.emit('pong', someid) - print 'cli_chan got:' - event = cli_chan.recv() - print event - assert event.name == 'pong' - assert event.args == someid - srv_chan.close() - cli_chan.close() - - def create_sub_multiplexer(events, from_event=None, - ignore_broadcast=False): - channel = events.channel(from_event) - sub_events = zerorpc.WrappedEvents(channel) - sub_multiplexer = zerorpc.ChannelMultiplexer(sub_events, - ignore_broadcast=ignore_broadcast) - return sub_multiplexer - - def open_sub_multiplexer(climux, srvmux): - someid = random.randint(0, 1000000) - print 'open...' - clisubmux = create_sub_multiplexer(climux, ignore_broadcast=True) - clisubmux.emit('open that', someid) - print 'srvsubmux got:' - event = srvmux.recv() - assert event.name == 'w' - srvsubmux = create_sub_multiplexer(srvmux, event) - event = srvsubmux.recv() - print event - return (clisubmux, srvsubmux) - - ping_pong(clientmux, servermux) - - (clientmux_lv2, servermux_lv2) = open_sub_multiplexer(clientmux, servermux) - ping_pong(clientmux_lv2, servermux_lv2) - - (clientmux_lv3, servermux_lv3) = open_sub_multiplexer(clientmux_lv2, - servermux_lv2) - ping_pong(clientmux_lv3, servermux_lv3) - - (clientmux_lv4, servermux_lv4) = open_sub_multiplexer(clientmux_lv3, - servermux_lv3) - ping_pong(clientmux_lv4, servermux_lv4) - - ping_pong(clientmux_lv4, servermux_lv4) - ping_pong(clientmux_lv3, servermux_lv3) - ping_pong(clientmux_lv2, servermux_lv2) - ping_pong(clientmux, servermux) - ping_pong(clientmux, servermux) - ping_pong(clientmux_lv2, servermux_lv2) - ping_pong(clientmux_lv4, servermux_lv4) - ping_pong(clientmux_lv3, servermux_lv3) - - (clientmux_lv5, servermux_lv5) = open_sub_multiplexer(clientmux_lv4, - servermux_lv4) - ping_pong(clientmux_lv5, servermux_lv5) diff --git a/zerorpc/events.py b/zerorpc/events.py index c358951..d2e6702 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -270,40 +270,3 @@ def setsockopt(self, *args): @property def context(self): return self._context - - -class WrappedEvents(object): - - def __init__(self, channel): - self._channel = channel - - def close(self): - pass - - @property - def recv_is_available(self): - return self._channel.recv_is_available - - def create_event(self, name, args, xheader=None): - xheader = {} if xheader is None else xheader - event = Event(name, args, self._channel.context) - event.header.update(xheader) - return event - - def emit_event(self, event, identity=None): - event_payload = (event.header, event.name, event.args) - wrapper_event = self._channel.create_event('w', event_payload) - self._channel.emit_event(wrapper_event) - - def emit(self, name, args, xheader=None): - wrapper_event = self.create_event(name, args, xheader) - self.emit_event(wrapper_event) - - def recv(self, timeout=None): - wrapper_event = self._channel.recv() - (header, name, args) = wrapper_event.args - return Event(name, args, None, header) - - @property - def context(self): - return self._channel.context From 36bcecd94ac652b22465aee9be5222f0c37294ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 01:34:10 -0700 Subject: [PATCH 036/113] Rename bufchan to channel After all, we are dealing with an abstract channel, no need to know if its buffered or not in the patterns. --- zerorpc/patterns.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index 526356c..60ad6e9 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -25,66 +25,66 @@ class ReqRep: - def process_call(self, context, bufchan, req_event, functor): + def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) result = functor(*req_event.args) - rep_event = bufchan.create_event('OK', (result,), + rep_event = channel.create_event('OK', (result,), context.hook_get_task_context()) context.hook_server_after_exec(req_event, rep_event) - bufchan.emit_event(rep_event) + channel.emit_event(rep_event) def accept_answer(self, event): return True - def process_answer(self, context, bufchan, req_event, rep_event, + def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): if rep_event.name == 'ERR': exception = handle_remote_error(rep_event) context.hook_client_after_request(req_event, rep_event, exception) raise exception context.hook_client_after_request(req_event, rep_event) - bufchan.close() + channel.close() result = rep_event.args[0] return result class ReqStream: - def process_call(self, context, bufchan, req_event, functor): + def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) xheader = context.hook_get_task_context() for result in iter(functor(*req_event.args)): - bufchan.emit('STREAM', result, xheader) - done_event = bufchan.create_event('STREAM_DONE', None, xheader) + channel.emit('STREAM', result, xheader) + done_event = channel.create_event('STREAM_DONE', None, xheader) # NOTE: "We" made the choice to call the hook once the stream is done, # the other choice was to call it at each iteration. I don't think that # one choice is better than the other, so I'm fine with changing this # or adding the server_after_iteration and client_after_iteration hooks. context.hook_server_after_exec(req_event, done_event) - bufchan.emit_event(done_event) + channel.emit_event(done_event) def accept_answer(self, event): return event.name in ('STREAM', 'STREAM_DONE') - def process_answer(self, context, bufchan, req_event, rep_event, + def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): def is_stream_done(rep_event): return rep_event.name == 'STREAM_DONE' - bufchan.on_close_if = is_stream_done + channel.on_close_if = is_stream_done def iterator(req_event, rep_event): while rep_event.name == 'STREAM': # Like in process_call, we made the choice to call the # after_exec hook only when the stream is done. yield rep_event.args - rep_event = bufchan.recv() + rep_event = channel.recv() if rep_event.name == 'ERR': exception = handle_remote_error(rep_event) context.hook_client_after_request(req_event, rep_event, exception) raise exception context.hook_client_after_request(req_event, rep_event) - bufchan.close() + channel.close() return iterator(req_event, rep_event) From 44dbb8927f66f8170ded223d96be72a36aa2bd1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:50:25 -0700 Subject: [PATCH 037/113] header -> xheader --- zerorpc/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zerorpc/core.py b/zerorpc/core.py index f30d076..045b186 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -405,9 +405,9 @@ def fork_task_context(functor, context=None): - if the new task will make any zerorpc call, it should be wrapped. ''' context = context or Context.get_instance() - header = context.hook_get_task_context() + xheader = context.hook_get_task_context() def wrapped(*args, **kargs): - context.hook_load_task_context(header) + context.hook_load_task_context(xheader) return functor(*args, **kargs) return wrapped From e62a705d464df21098123ada89d38c3e3fe8ca73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 01:29:23 -0700 Subject: [PATCH 038/113] Define a channel interface We are in python so it's merely a way to document the code. - requires a channel to accept timeouts on operations directly - expose few useful properties --- zerorpc/channel_base.py | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 zerorpc/channel_base.py diff --git a/zerorpc/channel_base.py b/zerorpc/channel_base.py new file mode 100644 index 0000000..a391b08 --- /dev/null +++ b/zerorpc/channel_base.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Open Source Initiative OSI - The MIT License (MIT):Licensing +# +# The MIT License (MIT) +# Copyright (c) 2014 François-Xavier Bourlet (bombela@gmail.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +class ChannelBase(object): + + @property + def context(self): + raise NotImplementedError() + + @property + def recv_is_supported(self): + raise NotImplementedError() + + @property + def emit_is_supported(self): + raise NotImplementedError() + + def close(self): + raise NotImplementedError() + + def new_event(self, name, args, xheader=None): + raise NotImplementedError() + + def emit_event(self, event, timeout=None): + raise NotImplementedError() + + def emit(self, name, args, xheader=None, timeout=None): + event = self.new_event(name, args, xheader) + return self.emit_event(event, timeout) + + def recv(self, timeout=None): + raise NotImplementedError() From d5efcebdaedaa4d46765e400160eb2e914084f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:05:24 -0700 Subject: [PATCH 039/113] Adapt events to the new channel API --- tests/test_events.py | 5 +- zerorpc/events.py | 106 +++++++++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 46 deletions(-) diff --git a/tests/test_events.py b/tests/test_events.py index 19ca208..bad2c8e 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -144,8 +144,9 @@ def test_events_dealer_router(): assert event.name == 'myevent' + str(i) assert list(event.args) == [i] - server.emit('answser' + str(i * 2), (i * 2,), - xheader=dict(zmqid=event.header['zmqid'])) + reply_event = server.new_event('answser' + str(i * 2), (i * 2,)) + reply_event.identity = event.identity + server.emit_event(reply_event) event = client.recv() print event assert event.name == 'answser' + str(i * 2) diff --git a/zerorpc/events.py b/zerorpc/events.py index d2e6702..2bca176 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -32,7 +32,9 @@ import gevent_zmq as zmq +from .exceptions import TimeoutExpired from .context import Context +from .channel_base import ChannelBase class Sender(object): @@ -64,8 +66,11 @@ def _sender(self): if not running: return - def __call__(self, parts): - self._send_queue.put(parts) + def __call__(self, parts, timeout=None): + try: + self._send_queue.put(parts, timeout=timeout) + except gevent.queue.Full: + raise TimeoutExpired(timeout) class Receiver(object): @@ -101,24 +106,25 @@ def _recver(self): break self._recv_queue.put(parts) - def __call__(self): - return self._recv_queue.get() + def __call__(self, timeout=None): + try: + return self._recv_queue.get(timeout=timeout) + except gevent.queue.Empty: + raise TimeoutExpired(timeout) class Event(object): - __slots__ = ['_name', '_args', '_header'] + __slots__ = ['_name', '_args', '_header', '_identity'] def __init__(self, name, args, context, header=None): self._name = name self._args = args if header is None: - self._header = { - 'message_id': context.new_msgid(), - 'v': 3 - } + self._header = {'message_id': context.new_msgid(), 'v': 3} else: self._header = header + self._identity = None @property def header(self): @@ -136,6 +142,14 @@ def name(self, v): def args(self): return self._args + @property + def identity(self): + return self._identity + + @identity.setter + def identity(self, v): + self._identity = v + def pack(self): return msgpack.Packer().pack((self._header, self._name, self._args)) @@ -164,27 +178,38 @@ def __str__(self, ignore_args=False): args = self._args try: args = '<<{0}>>'.format(str(self.unpack(self._args))) - except: + except Exception: pass - return '{0} {1} {2}'.format(self._name, self._header, - args) + if self._identity: + identity = ', '.join(repr(x) for x in self._identity) + return '<{0}> {1} {2} {3}'.format(identity, self._name, + self._header, args) + return '{0} {1} {2}'.format(self._name, self._header, args) -class Events(object): +class Events(ChannelBase): def __init__(self, zmq_socket_type, context=None): self._zmq_socket_type = zmq_socket_type self._context = context or Context.get_instance() - self._socket = zmq.Socket(self._context, zmq_socket_type) - self._send = self._socket.send_multipart - self._recv = self._socket.recv_multipart - if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.DEALER, zmq.ROUTER): + self._socket = self._context.socket(zmq_socket_type) + + if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.DEALER, zmq.ROUTER, zmq.REQ): self._send = Sender(self._socket) - if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER): + else: + self._send = None + + if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER, zmq.REP): self._recv = Receiver(self._socket) + else: + self._recv = None @property - def recv_is_available(self): - return self._zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER) + def recv_is_supported(self): + return self._recv is not None + + @property + def emit_is_supported(self): + return self._send is not None def __del__(self): try: @@ -226,42 +251,35 @@ def bind(self, endpoint, resolve=True): r.append(self._socket.bind(endpoint_)) return r - def create_event(self, name, args, xheader=None): - xheader = {} if xheader is None else xheader + def new_event(self, name, args, xheader=None): event = Event(name, args, context=self._context) - for k, v in xheader.items(): - if k == 'zmqid': - continue - event.header[k] = v + if xheader: + event.header.update(xheader) return event - def emit_event(self, event, identity=None): - if identity is not None: - parts = list(identity) + def emit_event(self, event, timeout=None): + if event.identity: + parts = list(event.identity or list()) parts.extend(['', event.pack()]) elif self._zmq_socket_type in (zmq.DEALER, zmq.ROUTER): parts = ('', event.pack()) else: parts = (event.pack(),) - self._send(parts) - - def emit(self, name, args, xheader=None): - xheader = {} if xheader is None else xheader - event = self.create_event(name, args, xheader) - identity = xheader.get('zmqid', None) - return self.emit_event(event, identity) + self._send(parts, timeout) - def recv(self): - parts = self._recv() - if len(parts) == 1: - identity = None - blob = parts[0] - else: + def recv(self, timeout=None): + parts = self._recv(timeout=timeout) + if len(parts) > 2: identity = parts[0:-2] blob = parts[-1] + elif len(parts) == 2: + identity = parts[0:-1] + blob = parts[-1] + else: + identity = None + blob = parts[0] event = Event.unpack(blob) - if identity is not None: - event.header['zmqid'] = identity + event.identity = identity return event def setsockopt(self, *args): From 69f3e9fc46fcdc58891b7e59ffa8fa3ab3705d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:44:46 -0700 Subject: [PATCH 040/113] Handle REP/REQ with new channel API --- zerorpc/events.py | 107 ++++++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 37 deletions(-) diff --git a/zerorpc/events.py b/zerorpc/events.py index 2bca176..2b0e396 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -37,34 +37,79 @@ from .channel_base import ChannelBase -class Sender(object): +class SequentialSender(object): + + def __init__(self, socket): + self._socket = socket + + def _send(self, parts): + e = None + for i in xrange(len(parts) - 1): + try: + self._socket.send(parts[i], copy=False, flags=zmq.SNDMORE) + except (gevent.GreenletExit, gevent.Timeout) as e: + if i == 0: + raise + self._socket.send(parts[i], copy=False, flags=zmq.SNDMORE) + try: + self._socket.send(parts[-1], copy=False) + except (gevent.GreenletExit, gevent.Timeout) as e: + self._socket.send(parts[-1], copy=False) + if e: + raise e + + def __call__(self, parts, timeout=None): + if timeout: + with gevent.Timeout(timeout): + self._send(parts) + else: + self._send(parts) + + +class SequentialReceiver(object): + + def __init__(self, socket): + self._socket = socket + + def _recv(self): + e = None + parts = [] + while True: + try: + part = self._socket.recv(copy=False) + except (gevent.GreenletExit, gevent.Timeout) as e: + if len(parts) == 0: + raise + part = self._socket.recv(copy=False) + parts.append(part) + if not part.more: + break + if e: + raise e + return parts + + def __call__(self, timeout=None): + if timeout: + with gevent.Timeout(timeout): + return self._recv() + else: + return self._recv() + + +class Sender(SequentialSender): def __init__(self, socket): self._socket = socket self._send_queue = gevent.queue.Channel() self._send_task = gevent.spawn(self._sender) - def __del__(self): - self.close() - def close(self): if self._send_task: self._send_task.kill() def _sender(self): - running = True for parts in self._send_queue: - for i in xrange(len(parts) - 1): - try: - self._socket.send(parts[i], flags=zmq.SNDMORE) - except gevent.GreenletExit: - if i == 0: - return - running = False - self._socket.send(parts[i], flags=zmq.SNDMORE) - self._socket.send(parts[-1]) - if not running: - return + super(Sender, self)._send(parts) def __call__(self, parts, timeout=None): try: @@ -73,37 +118,21 @@ def __call__(self, parts, timeout=None): raise TimeoutExpired(timeout) -class Receiver(object): +class Receiver(SequentialReceiver): def __init__(self, socket): self._socket = socket self._recv_queue = gevent.queue.Channel() self._recv_task = gevent.spawn(self._recver) - def __del__(self): - self.close() - def close(self): if self._recv_task: self._recv_task.kill() + self._recv_queue = None def _recver(self): - running = True while True: - parts = [] - while True: - try: - part = self._socket.recv() - except gevent.GreenletExit: - running = False - if len(parts) == 0: - return - part = self._socket.recv() - parts.append(part) - if not self._socket.getsockopt(zmq.RCVMORE): - break - if not running: - break + parts = super(Receiver, self)._recv() self._recv_queue.put(parts) def __call__(self, timeout=None): @@ -193,13 +222,17 @@ def __init__(self, zmq_socket_type, context=None): self._context = context or Context.get_instance() self._socket = self._context.socket(zmq_socket_type) - if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.DEALER, zmq.ROUTER, zmq.REQ): + if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.DEALER, zmq.ROUTER): self._send = Sender(self._socket) + elif zmq_socket_type in (zmq.REQ, zmq.REP): + self._send = SequentialSender(self._socket) else: self._send = None - if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER, zmq.REP): + if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER): self._recv = Receiver(self._socket) + elif zmq_socket_type in (zmq.REQ, zmq.REP): + self._recv = SequentialReceiver(self._socket) else: self._recv = None From 4bf37441a649f9154edf8c173532087a10bbbdd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:31:25 -0700 Subject: [PATCH 041/113] Adapt channel to the new channel API --- tests/test_channel.py | 18 ++++---- zerorpc/channel.py | 97 ++++++++++++++++++++----------------------- 2 files changed, 56 insertions(+), 59 deletions(-) diff --git a/tests/test_channel.py b/tests/test_channel.py index ed8367e..4d7ef6b 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -44,11 +44,12 @@ def test_events_channel_client_side(): event = server.recv() print event assert list(event.args) == [42] - assert event.header.get('zmqid', None) is not None + assert event.identity is not None - server.emit('someanswer', (21,), - xheader=dict(response_to=event.header['message_id'], - zmqid=event.header['zmqid'])) + reply_event = server.new_event('someanswer', (21,), + xheader=dict(response_to=event.header['message_id'])) + reply_event.identity = event.identity + server.emit_event(reply_event) event = client_channel.recv() assert list(event.args) == [21] @@ -69,12 +70,13 @@ def test_events_channel_client_side_server_send_many(): event = server.recv() print event assert list(event.args) == [10] - assert event.header.get('zmqid', None) is not None + assert event.identity is not None for x in xrange(10): - server.emit('someanswer', (x,), - xheader=dict(response_to=event.header['message_id'], - zmqid=event.header['zmqid'])) + reply_event = server.new_event('someanswer', (x,), + xheader=dict(response_to=event.header['message_id'])) + reply_event.identity = event.identity + server.emit_event(reply_event) for x in xrange(10): event = client_channel.recv() assert list(event.args) == [x] diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 5da8991..6c15bf5 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -29,26 +29,31 @@ import gevent.lock from .exceptions import TimeoutExpired +from .channel_base import ChannelBase from logging import getLogger logger = getLogger(__name__) -class ChannelMultiplexer(object): +class ChannelMultiplexer(ChannelBase): def __init__(self, events, ignore_broadcast=False): self._events = events self._active_channels = {} self._channel_dispatcher_task = None self._broadcast_queue = None - if events.recv_is_available and not ignore_broadcast: + if events.recv_is_supported and not ignore_broadcast: self._broadcast_queue = gevent.queue.Queue(maxsize=1) self._channel_dispatcher_task = gevent.spawn( self._channel_dispatcher) @property - def recv_is_available(self): - return self._events.recv_is_available + def recv_is_supported(self): + return self._events.recv_is_supported + + @property + def emit_is_supported(self): + return self._events.emit_is_supported def __del__(self): self.close() @@ -57,30 +62,25 @@ def close(self): if self._channel_dispatcher_task: self._channel_dispatcher_task.kill() - def create_event(self, name, args, xheader=None): - return self._events.create_event(name, args, xheader) - - def emit_event(self, event, identity=None): - return self._events.emit_event(event, identity) + def new_event(self, name, args, xheader=None): + return self._events.new_event(name, args, xheader) - def emit(self, name, args, xheader=None): - return self._events.emit(name, args, xheader) + def emit_event(self, event, timeout=None): + return self._events.emit_event(event, timeout) - def recv(self): + def recv(self, timeout=None): if self._broadcast_queue is not None: - event = self._broadcast_queue.get() + event = self._broadcast_queue.get(timeout=timeout) else: - event = self._events.recv() + event = self._events.recv(timeout=timeout) return event def _channel_dispatcher(self): while True: try: event = self._events.recv() - except Exception as e: - logger.error( - 'zerorpc.ChannelMultiplexer, ' - 'ignoring error on recv: {0}'.format(e)) + except Exception: + logger.exception('zerorpc.ChannelMultiplexer ignoring error on recv') continue channel_id = event.header.get('response_to', None) @@ -93,10 +93,9 @@ def _channel_dispatcher(self): queue = self._broadcast_queue if queue is None: - logger.error( - 'zerorpc.ChannelMultiplexer, ' - 'unable to route event: {0}' - .format(event.__str__(ignore_args=True))) + logger.warning('zerorpc.ChannelMultiplexer,' + ' unable to route event: {0}'.format( + event.__str__(ignore_args=True))) else: queue.put(event) @@ -115,7 +114,7 @@ def context(self): return self._events.context -class Channel(object): +class Channel(ChannelBase): def __init__(self, multiplexer, from_event=None): self._multiplexer = multiplexer @@ -124,13 +123,17 @@ def __init__(self, multiplexer, from_event=None): self._queue = gevent.queue.Queue(maxsize=1) if from_event is not None: self._channel_id = from_event.header['message_id'] - self._zmqid = from_event.header.get('zmqid', None) + self._zmqid = from_event.identity self._multiplexer._active_channels[self._channel_id] = self self._queue.put(from_event) @property - def recv_is_available(self): - return self._multiplexer.recv_is_available + def recv_is_supported(self): + return self._multiplexer.recv_is_supported + + @property + def emit_is_supported(self): + return self._multiplexer.emit_is_supported def __del__(self): self.close() @@ -140,21 +143,18 @@ def close(self): del self._multiplexer._active_channels[self._channel_id] self._channel_id = None - def create_event(self, name, args, xheader=None): - event = self._multiplexer.create_event(name, args, xheader) + def new_event(self, name, args, xheader=None): + event = self._multiplexer.new_event(name, args, xheader) if self._channel_id is None: self._channel_id = event.header['message_id'] self._multiplexer._active_channels[self._channel_id] = self else: event.header['response_to'] = self._channel_id + event.identity = self._zmqid return event - def emit(self, name, args, xheader=None): - event = self.create_event(name, args, xheader) - self._multiplexer.emit_event(event, self._zmqid) - - def emit_event(self, event): - self._multiplexer.emit_event(event, self._zmqid) + def emit_event(self, event, timeout=None): + self._multiplexer.emit_event(event, timeout) def recv(self, timeout=None): try: @@ -168,7 +168,7 @@ def context(self): return self._multiplexer.context -class BufferedChannel(object): +class BufferedChannel(ChannelBase): def __init__(self, channel, inqueue_size=100): self._channel = channel @@ -183,8 +183,12 @@ def __init__(self, channel, inqueue_size=100): self._recv_task = gevent.spawn(self._recver) @property - def recv_is_available(self): - return self._channel.recv_is_available + def recv_is_supported(self): + return self._channel.recv_is_supported + + @property + def emit_is_supported(self): + return self._channel.emit_is_supported @property def on_close_if(self): @@ -211,10 +215,8 @@ def _recver(self): if event.name == '_zpc_more': try: self._remote_queue_open_slots += int(event.args[0]) - except Exception as e: - logger.error( - 'gevent_zerorpc.BufferedChannel._recver, ' - 'exception: ' + repr(e)) + except Exception: + logger.exception('gevent_zerorpc.BufferedChannel._recver') if self._remote_queue_open_slots > 0: self._remote_can_recv.set() elif self._input_queue.qsize() == self._input_queue_size: @@ -227,13 +229,11 @@ def _recver(self): self.close() return - def create_event(self, name, args, xheader=None): - return self._channel.create_event(name, args, xheader) + def new_event(self, name, args, xheader=None): + return self._channel.new_event(name, args, xheader) - def emit_event(self, event, block=True, timeout=None): + def emit_event(self, event, timeout=None): if self._remote_queue_open_slots == 0: - if not block: - return False self._remote_can_recv.clear() self._remote_can_recv.wait(timeout=timeout) self._remote_queue_open_slots -= 1 @@ -242,11 +242,6 @@ def emit_event(self, event, block=True, timeout=None): except: self._remote_queue_open_slots += 1 raise - return True - - def emit(self, name, args, xheader=None, block=True, timeout=None): - event = self.create_event(name, args, xheader) - return self.emit_event(event, block, timeout) def _request_data(self): open_slots = self._input_queue_size - self._input_queue_reserved From 6997d8157414e45c8cb44be44d621bc8fd5fc74b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:28:57 -0700 Subject: [PATCH 042/113] Adapt heartbeat to the new channel API --- zerorpc/heartbeat.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index c4e7c4b..e2cd040 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -31,9 +31,10 @@ import gevent.lock from .exceptions import * # noqa +from .channel_base import ChannelBase -class HeartBeatOnChannel(object): +class HeartBeatOnChannel(ChannelBase): def __init__(self, channel, freq=5, passive=False): self._channel = channel @@ -49,8 +50,12 @@ def __init__(self, channel, freq=5, passive=False): self._start_heartbeat() @property - def recv_is_available(self): - return self._channel.recv_is_available + def recv_is_supported(self): + return self._channel.recv_is_supported + + @property + def emit_is_supported(self): + return self._channel.emit_is_supported def __del__(self): self.close() @@ -100,31 +105,24 @@ def _lost_remote_exception(self): return LostRemote('Lost remote after {0}s heartbeat'.format( self._heartbeat_freq * 2)) - def create_event(self, name, args, xheader=None): + def new_event(self, name, args, header=None): if self._compat_v2 and name == '_zpc_more': name = '_zpc_hb' - return self._channel.create_event(name, args, xheader) + return self._channel.new_event(name, args, header) - def emit_event(self, event): + def emit_event(self, event, timeout=None): if self._lost_remote: raise self._lost_remote_exception() - self._channel.emit_event(event) - - def emit(self, name, args, xheader=None): - event = self.create_event(name, args, xheader) - self.emit_event(event) + self._channel.emit_event(event, timeout) def recv(self, timeout=None): if self._lost_remote: raise self._lost_remote_exception() - try: - event = self._input_queue.get(timeout=timeout) + return self._input_queue.get(timeout=timeout) except gevent.queue.Empty: raise TimeoutExpired(timeout) - return event - @property def channel(self): return self._channel From 74217a73808d75f3052ad0e0c2fcf32411bb6813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:27:15 -0700 Subject: [PATCH 043/113] Adapt core to new channel API --- zerorpc/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zerorpc/core.py b/zerorpc/core.py index 045b186..b902ecd 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -152,7 +152,7 @@ def _async_task(self, initial_event): except Exception: exc_infos = list(sys.exc_info()) human_exc_infos = self._print_traceback(protocol_v1, exc_infos) - reply_event = bufchan.create_event('ERR', human_exc_infos, + reply_event = bufchan.new_event('ERR', human_exc_infos, self._context.hook_get_task_context()) self._context.hook_server_inspect_exception(event, reply_event, exc_infos) bufchan.emit_event(reply_event) @@ -236,7 +236,7 @@ def __call__(self, method, *args, **kargs): bufchan = BufferedChannel(hbchan, inqueue_size=kargs.get('slots', 100)) xheader = self._context.hook_get_task_context() - request_event = bufchan.create_event(method, args, xheader) + request_event = bufchan.new_event(method, args, xheader) self._context.hook_client_before_request(request_event) bufchan.emit_event(request_event) From dc55b6a36ed949750e3382ca4003b98140472a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 01:56:50 -0700 Subject: [PATCH 044/113] Adapt the patterns to the new channel API --- zerorpc/patterns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index 60ad6e9..d8872f9 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -28,7 +28,7 @@ class ReqRep: def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) result = functor(*req_event.args) - rep_event = channel.create_event('OK', (result,), + rep_event = channel.new_event('OK', (result,), context.hook_get_task_context()) context.hook_server_after_exec(req_event, rep_event) channel.emit_event(rep_event) @@ -55,7 +55,7 @@ def process_call(self, context, channel, req_event, functor): xheader = context.hook_get_task_context() for result in iter(functor(*req_event.args)): channel.emit('STREAM', result, xheader) - done_event = channel.create_event('STREAM_DONE', None, xheader) + done_event = channel.new_event('STREAM_DONE', None, xheader) # NOTE: "We" made the choice to call the hook once the stream is done, # the other choice was to call it at each iteration. I don't think that # one choice is better than the other, so I'm fine with changing this From a100a5a27c0eccd4a769b5a45068889082c8dc23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:24:01 -0700 Subject: [PATCH 045/113] Channels are closed responsibly by patterns Instead of doing horrible __del__ stuff and freezing everything in the gc, and sometimes getting totally absurd errors: -> lets cleanly and explicitly close our channels all the way down. --- zerorpc/channel.py | 9 --------- zerorpc/core.py | 20 ++++++-------------- zerorpc/heartbeat.py | 3 --- zerorpc/patterns.py | 41 ++++++++++++++++++++++------------------- 4 files changed, 28 insertions(+), 45 deletions(-) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 6c15bf5..b2af984 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -55,9 +55,6 @@ def recv_is_supported(self): def emit_is_supported(self): return self._events.emit_is_supported - def __del__(self): - self.close() - def close(self): if self._channel_dispatcher_task: self._channel_dispatcher_task.kill() @@ -135,9 +132,6 @@ def recv_is_supported(self): def emit_is_supported(self): return self._multiplexer.emit_is_supported - def __del__(self): - self.close() - def close(self): if self._channel_id is not None: del self._multiplexer._active_channels[self._channel_id] @@ -198,9 +192,6 @@ def on_close_if(self): def on_close_if(self, cb): self._on_close_if = cb - def __del__(self): - self.close() - def close(self): if self._recv_task is not None: self._recv_task.kill() diff --git a/zerorpc/core.py b/zerorpc/core.py index b902ecd..29f0d45 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -240,21 +240,13 @@ def __call__(self, method, *args, **kargs): self._context.hook_client_before_request(request_event) bufchan.emit_event(request_event) - try: - if kargs.get('async', False) is False: - return self._process_response(request_event, bufchan, timeout) + if kargs.get('async', False) is False: + return self._process_response(request_event, bufchan, timeout) - async_result = gevent.event.AsyncResult() - gevent.spawn(self._process_response, request_event, bufchan, - timeout).link(async_result) - return async_result - except: - # XXX: This is going to be closed twice if async is false and - # _process_response raises an exception. I wonder if the above - # async branch can raise an exception too, if no we can just remove - # this code. - bufchan.close() - raise + async_result = gevent.event.AsyncResult() + gevent.spawn(self._process_response, request_event, bufchan, + timeout).link(async_result) + return async_result def __getattr__(self, method): return lambda *args, **kargs: self(method, *args, **kargs) diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index e2cd040..69205f8 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -57,9 +57,6 @@ def recv_is_supported(self): def emit_is_supported(self): return self._channel.emit_is_supported - def __del__(self): - self.close() - def close(self): if self._heartbeat_task is not None: self._heartbeat_task.kill() diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index d8872f9..08dc1d6 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -38,14 +38,15 @@ def accept_answer(self, event): def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): - if rep_event.name == 'ERR': - exception = handle_remote_error(rep_event) - context.hook_client_after_request(req_event, rep_event, exception) - raise exception - context.hook_client_after_request(req_event, rep_event) - channel.close() - result = rep_event.args[0] - return result + try: + if rep_event.name == 'ERR': + exception = handle_remote_error(rep_event) + context.hook_client_after_request(req_event, rep_event, exception) + raise exception + context.hook_client_after_request(req_event, rep_event) + return rep_event.args[0] + finally: + channel.close() class ReqStream: @@ -74,17 +75,19 @@ def is_stream_done(rep_event): channel.on_close_if = is_stream_done def iterator(req_event, rep_event): - while rep_event.name == 'STREAM': - # Like in process_call, we made the choice to call the - # after_exec hook only when the stream is done. - yield rep_event.args - rep_event = channel.recv() - if rep_event.name == 'ERR': - exception = handle_remote_error(rep_event) - context.hook_client_after_request(req_event, rep_event, exception) - raise exception - context.hook_client_after_request(req_event, rep_event) - channel.close() + try: + while rep_event.name == 'STREAM': + # Like in process_call, we made the choice to call the + # after_exec hook only when the stream is done. + yield rep_event.args + rep_event = channel.recv() + if rep_event.name == 'ERR': + exception = handle_remote_error(rep_event) + context.hook_client_after_request(req_event, rep_event, exception) + raise exception + context.hook_client_after_request(req_event, rep_event) + finally: + channel.close() return iterator(req_event, rep_event) From 59327237b53a6fa95a35e4be738d3890ab5f5a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 01:56:03 -0700 Subject: [PATCH 046/113] ReqRep pattern will only accept OK and ERR events. This mostly for pretty symmetry with the ReqStream. --- zerorpc/patterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index 08dc1d6..e944e44 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -34,7 +34,7 @@ def process_call(self, context, channel, req_event, functor): channel.emit_event(rep_event) def accept_answer(self, event): - return True + return event.name in ('OK', 'ERR') def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): From 049c0b3e83efad936e4e5063080325202afeadb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 02:18:05 -0700 Subject: [PATCH 047/113] Open pandora box: allow hooks for patterns --- zerorpc/context.py | 8 +++++++- zerorpc/core.py | 32 ++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/zerorpc/context.py b/zerorpc/context.py index 879e6c4..ef8bf50 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -44,7 +44,8 @@ def __init__(self): 'server_inspect_exception': [], 'client_handle_remote_error': [], 'client_before_request': [], - 'client_after_request': [] + 'client_after_request': [], + 'client_patterns_list': [], } self._reset_msgid() @@ -217,3 +218,8 @@ def hook_client_after_request(self, request_event, reply_event, exception=None): """ for functor in self._hooks['client_after_request']: functor(request_event, reply_event, exception) + + def hook_client_patterns_list(self, patterns): + for functor in self._hooks['client_patterns_list']: + patterns = functor(patterns) + return patterns diff --git a/zerorpc/core.py b/zerorpc/core.py index 29f0d45..04185a8 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -206,27 +206,31 @@ def _handle_remote_error(self, event): return exception def _select_pattern(self, event): - for pattern in patterns.patterns_list: + for pattern in self._context.hook_client_patterns_list( + patterns.patterns_list): if pattern.accept_answer(event): return pattern - msg = 'Unable to find a pattern for: {0}'.format(event) - raise RuntimeError(msg) + return None def _process_response(self, request_event, bufchan, timeout): - try: - reply_event = bufchan.recv(timeout) - pattern = self._select_pattern(reply_event) - return pattern.process_answer(self._context, bufchan, request_event, - reply_event, self._handle_remote_error) - except TimeoutExpired: + def raise_error(ex): bufchan.close() - ex = TimeoutExpired(timeout, - 'calling remote method {0}'.format(request_event.name)) self._context.hook_client_after_request(request_event, None, ex) raise ex - except: - bufchan.close() - raise + + try: + reply_event = bufchan.recv(timeout=timeout) + except TimeoutExpired: + raise_error(TimeoutExpired(timeout, + 'calling remote method {0}'.format(request_event.name))) + + pattern = self._select_pattern(reply_event) + if pattern is None: + raise_error(RuntimeError( + 'Unable to find a pattern for: {0}'.format(request_event))) + + return pattern.process_answer(self._context, bufchan, request_event, + reply_event, self._handle_remote_error) def __call__(self, method, *args, **kargs): timeout = kargs.get('timeout', self._timeout) From 621af25427a7f29e616f6879dda168dd06dab95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 22:44:59 -0700 Subject: [PATCH 048/113] More robust unit tests, less race conditions - Bump timouts value to avoid false unit test errors: Basically, some timeouts are a little bit to short for fast testing. - Fix transiant unit test failure on pubsub. - you log all msgs - you send one msg - heartbeat msgs are logged - you check for only one msg: bad - you check if your msg appars among the heartbeats: good This was a beautiful race condition, since most of the time, the heartbeat is slower than how fast you send and test your msg. - Wrap in coroutines. HeartbeatOnChannel can raises to its parent at any time. Encapsulating the code in its own coroutine will reduce race conditions. --- tests/test_buffered_channel.py | 73 +++++++++++++++++++++------------- tests/test_client_heartbeat.py | 11 ++--- tests/test_heartbeat.py | 4 +- tests/test_middleware.py | 14 +++---- tests/test_reqstream.py | 4 +- 5 files changed, 61 insertions(+), 45 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index 93c7db2..8152bca 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -149,22 +149,28 @@ def test_heartbeat_can_open_channel_client_close(): client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) + def server_fn(): + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan) + try: + while True: + gevent.sleep(1) + finally: + server_bufchan.close() + server_coro = gevent.spawn(server_fn) gevent.sleep(TIME_FACTOR * 3) print 'CLOSE CLIENT SOCKET!!!' client_bufchan.close() client.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_bufchan.recv) + assert_raises(zerorpc.LostRemote, server_coro.get()) else: with assert_raises(zerorpc.LostRemote): - client_bufchan.recv() + server_coro.get() print 'SERVER LOST CLIENT :)' - server_bufchan.close() server.close() @@ -178,16 +184,11 @@ def test_do_some_req_rep(): client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) - client_bufchan = zerorpc.BufferedChannel(client_hbchan) - - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) def client_do(): + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan) for x in xrange(20): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() @@ -199,6 +200,11 @@ def client_do(): coro_pool.spawn(client_do) def server_do(): + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan) + for x in xrange(20): event = server_bufchan.recv() assert event.name == 'add' @@ -389,18 +395,22 @@ def test_congestion_control_server_pushing(): client_channel = client.channel() client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) - client_bufchan = zerorpc.BufferedChannel(client_hbchan) + client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=100) event = server.recv() server_channel = server.channel(event) server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) + server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=100) + + read_cnt = 0 def client_do(): for x in xrange(200): event = client_bufchan.recv() assert event.name == 'coucou' assert event.args == x + read_cnt += 1 + raise Exception('CLIENT DONE') coro_pool = gevent.pool.Pool() coro_pool.spawn(client_do) @@ -409,30 +419,37 @@ def server_do(): if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(200): - server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 1 + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: with assert_raises(zerorpc.TimeoutExpired): for x in xrange(200): - server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 1 + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 server_bufchan.emit('coucou', 1) # block until receiver is ready if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(2, 200): - server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 100 + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: with assert_raises(zerorpc.TimeoutExpired): for x in xrange(2, 200): - server_bufchan.emit('coucou', x, timeout=TIME_FACTOR * 0) # will fail when x == 100 - for x in xrange(101, 200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 + for x in xrange(read_cnt, 200): server_bufchan.emit('coucou', x) # block until receiver is ready + raise Exception('SERVER DONE') coro_pool.spawn(server_do) - - coro_pool.join() - client_bufchan.close() - client.close() - server_bufchan.close() - server.close() + try: + coro_pool.join() + except zerorpc.LostRemote: + pass + finally: + try: + client_bufchan.close() + client.close() + server_bufchan.close() + server.close() + except Exception: + pass diff --git a/tests/test_client_heartbeat.py b/tests/test_client_heartbeat.py index d884aee..ccabe32 100644 --- a/tests/test_client_heartbeat.py +++ b/tests/test_client_heartbeat.py @@ -60,12 +60,12 @@ def lolita(self): gevent.sleep(TIME_FACTOR * 3) return 42 - srv = MySrv(heartbeat=TIME_FACTOR * 1) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) - gevent.sleep(TIME_FACTOR * 0) + gevent.sleep(0) - client = zerorpc.Client(heartbeat=TIME_FACTOR * 1) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4) client.connect(endpoint) assert client.lolita() == 42 @@ -84,11 +84,12 @@ def slow(self): gevent.sleep(TIME_FACTOR * 3) return 2 - srv = MySrv(heartbeat=TIME_FACTOR * 1) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) + gevent.sleep(0) - client = zerorpc.Client(heartbeat=TIME_FACTOR * 1, passive_heartbeat=True) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4, passive_heartbeat=True) client.connect(endpoint) assert client.slow() == 2 diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index f6234f7..756e413 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -171,11 +171,11 @@ def test_do_some_req_rep(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 4) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 4) def client_do(): for x in xrange(20): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index c1bb073..8ba9d2f 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -373,20 +373,18 @@ def echo(self, msg): trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). - for attempt in xrange(0, 100): + while not trigger.is_set(): c.echo('pub...') - if trigger.wait(TIME_FACTOR * 0.2): + if trigger.wait(TIME_FACTOR * 1): break subscriber.stop() subscriber_task.join() - assert publisher_tracer._log == [ - ('new', publisher_tracer.trace_id), - ] - assert subscriber_tracer._log == [ - ('load', publisher_tracer.trace_id), - ] + print publisher_tracer._log + assert ('new', publisher_tracer.trace_id) in publisher_tracer._log + print subscriber_tracer._log + assert ('load', publisher_tracer.trace_id) in subscriber_tracer._log class InspectExceptionMiddleware(Tracer): diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index a06c4f3..925d470 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -42,11 +42,11 @@ def range(self, max): def xrange(self, max): return xrange(max) - srv = MySrv(heartbeat=TIME_FACTOR * 2) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=TIME_FACTOR * 2) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4) client.connect(endpoint) r = client.range(10) From 3056243b942d14dd62c6e8ae0d9e7847b3999298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 00:47:43 -0700 Subject: [PATCH 049/113] Bump pyzmq requirement to >=13.1.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 13d4149..a27a954 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ requirements = [ 'gevent>=1.0', 'msgpack-python', - 'pyzmq==13.1.0' + 'pyzmq>=13.1.0' ] if sys.version_info < (2, 7): requirements.append('argparse') From 2bb54278af97a5513119a8c1c681b3ec5c1d73e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 9 Sep 2014 00:40:38 -0700 Subject: [PATCH 050/113] Do not ignore test errors with pyzqm>=14 on travis. --- .travis.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index d162685..a4f2163 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,13 @@ python: - 2.7 env: matrix: - - PYZMQ='pyzmq==14.1.1' - - PYZMQ='pyzmq==14.1.0' - - PYZMQ='pyzmq==14.0.1' - - PYZMQ='pyzmq==14.0.0' - - PYZMQ='pyzmq==13.1.0' - - PYZMQ='pyzmq==13.0.2' - - PYZMQ='pyzmq==13.0.0' + - PYZMQ='pyzmq>=14.3' + - PYZMQ='pyzmq>=14.2,<14.3' + - PYZMQ='pyzmq>=14.1,<14.2' + - PYZMQ='pyzmq>=14.0,<14.1' + - PYZMQ='pyzmq<14' matrix: fast_finish: true - allow_failures: - - env: PYZMQ='pyzmq==14.0.0' - - env: PYZMQ='pyzmq==14.0.1' - - env: PYZMQ='pyzmq==14.1.0' - - env: PYZMQ='pyzmq==14.1.1' script: - flake8 --ignore=E501,E128 zerorpc bin - ZPC_TEST_TIME_FACTOR=0.1 nosetests From 22e97500c9a2e7c8eed20af1f51611cdc1a939fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 12 Sep 2014 22:12:42 -0700 Subject: [PATCH 051/113] test_zmq: use unix socket instead of tcp Simply avoid binding to an already used port. --- tests/test_zmq.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_zmq.py b/tests/test_zmq.py index 6e9dad1..36104a2 100644 --- a/tests/test_zmq.py +++ b/tests/test_zmq.py @@ -26,13 +26,15 @@ import gevent from zerorpc import zmq +from testutils import random_ipc_endpoint def test1(): + endpoint = random_ipc_endpoint() def server(): c = zmq.Context() s = c.socket(zmq.REP) - s.bind('tcp://0.0.0.0:9999') + s.bind(endpoint) while True: print 'srv recving...' r = s.recv() @@ -46,7 +48,7 @@ def server(): def client(): c = zmq.Context() s = c.socket(zmq.REQ) - s.connect('tcp://localhost:9999') + s.connect(endpoint) print 'cli sending...' s.send('hello') From d913118f0ad4117ddb78f81427c8e80c44788d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 12 Sep 2014 22:48:27 -0700 Subject: [PATCH 052/113] Print correctly zmq identity when debugging. --- zerorpc/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/events.py b/zerorpc/events.py index 2b0e396..ae30bb6 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -210,7 +210,7 @@ def __str__(self, ignore_args=False): except Exception: pass if self._identity: - identity = ', '.join(repr(x) for x in self._identity) + identity = ', '.join(repr(x.bytes) for x in self._identity) return '<{0}> {1} {2} {3}'.format(identity, self._name, self._header, args) return '{0} {1} {2}'.format(self._name, self._header, args) From 8c612a2756ef0a65d9a5404b7fce27b33f6b0fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 12 Sep 2014 22:56:18 -0700 Subject: [PATCH 053/113] Fix flickering unit test HeartBeatOnChannel raises exceptions directly to its parent's coroutine so it needs to run in the same coroutine as the test server/client. --- tests/test_buffered_channel.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index 8152bca..746174c 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -393,29 +393,28 @@ def test_congestion_control_server_pushing(): client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) - client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=100) - - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=100) - read_cnt = 0 def client_do(): + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=100) for x in xrange(200): event = client_bufchan.recv() assert event.name == 'coucou' assert event.args == x + global read_cnt read_cnt += 1 - raise Exception('CLIENT DONE') + client_bufchan.close() coro_pool = gevent.pool.Pool() coro_pool.spawn(client_do) def server_do(): + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=100) if sys.version_info < (2, 7): def _do_with_assert_raises(): for x in xrange(200): @@ -437,8 +436,7 @@ def _do_with_assert_raises(): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 for x in xrange(read_cnt, 200): server_bufchan.emit('coucou', x) # block until receiver is ready - raise Exception('SERVER DONE') - + server_bufchan.close() coro_pool.spawn(server_do) try: @@ -446,10 +444,5 @@ def _do_with_assert_raises(): except zerorpc.LostRemote: pass finally: - try: - client_bufchan.close() - client.close() - server_bufchan.close() - server.close() - except Exception: - pass + client.close() + server.close() From 5c66bf083acba8fac71dfba686b87da673c5457e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 12 Sep 2014 23:50:10 -0700 Subject: [PATCH 054/113] Fix a really stupid and old bug. - server side, you accept a new channel - you stack an heartbeatonchannel on top - you slow everything enough so the client already sent some heartbeat msgs before you could read the first msg (cpu intensive task, time.sleep...) - you send back the response - you close the heartbeatonchannel - this kills the coroutine handling the heartbeat - this action eventually switch to the hearbeat coroutine to raise GreenletExit in it. - which after might ends up to the recver coroutine (not yet killed) - the recever coroutine gets the heartbeat msg from the client - and stupidly process to spawn a new heartbeat coroutine! - this eventually switch back to the close() - which set the self._heartbeat to None (so potentially another heartbeat msg could spawn another heartbeat coroutine!) - after enough back and forth, self._channel is set to None - one or more of the heartbeat coroutines is trying to access self._channel.emit... and BOOM, you get (an harmless) stacktrace in the logs. The solution is to set a flag when close() is called: self._close = True. _start_hearbeat() called from the recver coroutine can then check this flag. Last but not least, the heartbeat coroutine checks this flag before killing the parent coroutine. This is not strictly necessary, but if in the future the code in close() eventually yields back to gevent (for example, by trying to kill another coroutine before the heartbeat's one), this will prevent a LostRemote to be raised on close(). And finally, a forgotten variable was removed (in channel.py). --- zerorpc/channel.py | 1 - zerorpc/heartbeat.py | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index b2af984..9d4b196 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -171,7 +171,6 @@ def __init__(self, channel, inqueue_size=100): self._input_queue_reserved = 1 self._remote_can_recv = gevent.event.Event() self._input_queue = gevent.queue.Queue() - self._lost_remote = False self._verbose = False self._on_close_if = None self._recv_task = gevent.spawn(self._recver) diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index 69205f8..48c48b1 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -37,6 +37,7 @@ class HeartBeatOnChannel(ChannelBase): def __init__(self, channel, freq=5, passive=False): + self._closed = False self._channel = channel self._heartbeat_freq = freq self._input_queue = gevent.queue.Channel() @@ -58,6 +59,7 @@ def emit_is_supported(self): return self._channel.emit_is_supported def close(self): + self._closed = True if self._heartbeat_task is not None: self._heartbeat_task.kill() self._heartbeat_task = None @@ -75,13 +77,14 @@ def _heartbeat(self): self._remote_last_hb = time.time() if time.time() > self._remote_last_hb + self._heartbeat_freq * 2: self._lost_remote = True - gevent.kill(self._parent_coroutine, - self._lost_remote_exception()) + if not self._closed: + gevent.kill(self._parent_coroutine, + self._lost_remote_exception()) break self._channel.emit('_zpc_hb', (0,)) # 0 -> compat with protocol v2 def _start_heartbeat(self): - if self._heartbeat_task is None and self._heartbeat_freq is not None: + if self._heartbeat_task is None and self._heartbeat_freq is not None and not self._closed: self._heartbeat_task = gevent.spawn(self._heartbeat) def _recver(self): From 850e3302b2a8c8dce2cc8096491680ff50237dcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 12 Sep 2014 23:54:32 -0700 Subject: [PATCH 055/113] --debug for printing logging events passing by. - zerorpc cli accepts --debug or -d, this turn on DEBUG level logging and active low level events debugging on the zerorpc object. - a zerorpc.SocketBase exposes a new boolean property debug --- zerorpc/channel.py | 4 ++++ zerorpc/cli.py | 10 ++++++++++ zerorpc/events.py | 22 +++++++++++++++++++++- zerorpc/socket.py | 8 ++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 9d4b196..136d3a7 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -27,6 +27,7 @@ import gevent.event import gevent.local import gevent.lock +import logging from .exceptions import TimeoutExpired from .channel_base import ChannelBase @@ -122,6 +123,7 @@ def __init__(self, multiplexer, from_event=None): self._channel_id = from_event.header['message_id'] self._zmqid = from_event.identity self._multiplexer._active_channels[self._channel_id] = self + logging.debug('<-- new channel %s', self._channel_id) self._queue.put(from_event) @property @@ -135,6 +137,7 @@ def emit_is_supported(self): def close(self): if self._channel_id is not None: del self._multiplexer._active_channels[self._channel_id] + logging.debug('-x- closed channel %s', self._channel_id) self._channel_id = None def new_event(self, name, args, xheader=None): @@ -142,6 +145,7 @@ def new_event(self, name, args, xheader=None): if self._channel_id is None: self._channel_id = event.header['message_id'] self._multiplexer._active_channels[self._channel_id] = self + logging.debug('--> new channel %s', self._channel_id) else: event.header['response_to'] = self._channel_id event.identity = self._zmqid diff --git a/zerorpc/cli.py b/zerorpc/cli.py index b542ad7..b81a55e 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -69,6 +69,9 @@ parser.add_argument('--active-hb', default=False, action='store_true', help='enable active heartbeat. The default is to \ wait for the server to send the first heartbeat') +parser.add_argument('-d', '--debug', default=False, action='store_true', + help='Print zerorpc debug msgs, \ + like outgoing and incomming messages.') parser.add_argument('address', nargs='?', help='address to connect to. Skip \ this if you specified --connect or --bind at least once') parser.add_argument('command', nargs='?', @@ -110,6 +113,8 @@ def run_server(args): server_obj = server_obj() server = zerorpc.Server(server_obj, heartbeat=args.heartbeat) + if args.debug: + server.debug = True setup_links(args, server) print 'serving "{0}"'.format(server_obj_path) return server.run() @@ -209,6 +214,8 @@ def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): def run_client(args): client = zerorpc.Client(timeout=args.timeout, heartbeat=args.heartbeat, passive_heartbeat=not args.active_hb) + if args.debug: + client.debug = True setup_links(args, client) if not args.command: (longest_name_len, detailled_methods) = zerorpc_inspect(client, @@ -257,6 +264,9 @@ def main(): logging.basicConfig() args = parser.parse_args() + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + if args.bind or args.connect: if args.command: args.params.insert(0, args.command) diff --git a/zerorpc/events.py b/zerorpc/events.py index ae30bb6..8c6c7f6 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -29,7 +29,7 @@ import gevent.event import gevent.local import gevent.lock - +import logging import gevent_zmq as zmq from .exceptions import TimeoutExpired @@ -218,6 +218,7 @@ def __str__(self, ignore_args=False): class Events(ChannelBase): def __init__(self, zmq_socket_type, context=None): + self._debug = False self._zmq_socket_type = zmq_socket_type self._context = context or Context.get_instance() self._socket = self._context.socket(zmq_socket_type) @@ -262,6 +263,19 @@ def close(self): pass self._socket.close() + @property + def debug(self): + return self._debug + + @debug.setter + def debug(self, v): + if v != self._debug: + self._debug = v + if self._debug: + logging.debug('debug enabled') + else: + logging.debug('debug disabled') + def _resolve_endpoint(self, endpoint, resolve=True): if resolve: endpoint = self._context.hook_resolve_endpoint(endpoint) @@ -276,12 +290,14 @@ def connect(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.connect(endpoint_)) + logging.debug('connected to %s (status=%s)', endpoint_, r[-1]) return r def bind(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.bind(endpoint_)) + logging.debug('bound to %s (status=%s)', endpoint_, r[-1]) return r def new_event(self, name, args, xheader=None): @@ -291,6 +307,8 @@ def new_event(self, name, args, xheader=None): return event def emit_event(self, event, timeout=None): + if self._debug: + logging.debug('--> %s', event) if event.identity: parts = list(event.identity or list()) parts.extend(['', event.pack()]) @@ -313,6 +331,8 @@ def recv(self, timeout=None): blob = parts[0] event = Event.unpack(blob) event.identity = identity + if self._debug: + logging.debug('<-- %s', event) return event def setsockopt(self, *args): diff --git a/zerorpc/socket.py b/zerorpc/socket.py index 2a7020e..284febe 100644 --- a/zerorpc/socket.py +++ b/zerorpc/socket.py @@ -41,3 +41,11 @@ def connect(self, endpoint, resolve=True): def bind(self, endpoint, resolve=True): return self._events.bind(endpoint, resolve) + + @property + def debug(self): + return self._events.debug + + @debug.setter + def debug(self, v): + self._events.debug = v From e3b3ebb282d445d0673c391a65f77d24e3f577bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sat, 13 Sep 2014 01:01:00 -0700 Subject: [PATCH 056/113] Guess better the name of a service. --- zerorpc/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zerorpc/core.py b/zerorpc/core.py index 04185a8..6758cff 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -81,7 +81,9 @@ def _filter_methods(cls, self, methods): @staticmethod def _extract_name(methods): - return getattr(type(methods), '__name__', None) or repr(methods) + return getattr(methods, '__name__', None) \ + or getattr(type(methods), '__name__', None) \ + or repr(methods) def close(self): self.stop() From fdac478e327aafceec3009c80097cf49117c7027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sat, 13 Sep 2014 01:02:02 -0700 Subject: [PATCH 057/113] cli remote inspection: handle empty results better --- zerorpc/cli.py | 59 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/zerorpc/cli.py b/zerorpc/cli.py index b81a55e..abf6edd 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -147,12 +147,12 @@ def remote_detailled_methods(): r = [(name + (inspect.formatargspec(*argspec) if argspec else '(...)'), doc) for name, argspec, doc in remote_detailled_methods()] - longest_name_len = max(len(name) for name, doc in r) + longest_name_len = max(len(name) for name, doc in r) if r else 0 return (longest_name_len, r) # handle the 'python formatted' _zerorpc_inspect, that return the output of -# "getargspec" from the python lib "inspect". +# "getargspec" from the python lib "inspect". A monstruosity from protocol v2. def zerorpc_inspect_python_argspecs(remote_methods, filter_method, long_doc, include_argspec): def format_method(name, argspec, doc): if include_argspec: @@ -165,10 +165,13 @@ def format_method(name, argspec, doc): return (name, doc) r = [format_method(*methods_info) for methods_info in remote_methods if filter_method is None or methods_info[0] == filter_method] - longest_name_len = max(len(name) for name, doc in r) + if not r: + return None + longest_name_len = max(len(name) for name, doc in r) if r else 0 return (longest_name_len, r) +# Handles generically formatted arguments (not tied to any specific programming language). def zerorpc_inspect_generic(remote_methods, filter_method, long_doc, include_argspec): def format_method(name, args, doc): if include_argspec: @@ -178,37 +181,53 @@ def format_arg(arg): return arg['name'] return '{0}={1}'.format(arg['name'], def_val) - name += '({0})'.format(', '.join(map(format_arg, args))) + if args: + name += '({0})'.format(', '.join(map(format_arg, args))) + else: + name += '(??)' + if not doc: doc = '' elif not long_doc: doc = doc.splitlines()[0] return (name, doc) - methods = [format_method(name, details['args'], details['doc']) for name, details in remote_methods.items() + methods = [format_method(name, details['args'], details['doc']) + for name, details in remote_methods.items() if filter_method is None or name == filter_method] - longest_name_len = max(len(name) for name, doc in methods) + longest_name_len = (max(len(name) for name, doc in methods) + if methods else 0) return (longest_name_len, methods) def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): try: - remote_methods = client._zerorpc_inspect()['methods'] + inspect_result = client._zerorpc_inspect() + remote_methods = inspect_result['methods'] legacy = False except (zerorpc.RemoteError, NameError): legacy = True if legacy: - return zerorpc_inspect_legacy(client, method, - long_doc, include_argspec) + try: + service_name = client._zerorpc_name() + except (zerorpc.RemoteError): + service_name = 'N/A' - if not isinstance(remote_methods, dict): - return zerorpc_inspect_python_argspecs(remote_methods, method, long_doc, - include_argspec) + (longest_name_len, detailled_methods) = zerorpc_inspect_legacy(client, + method, long_doc, include_argspec) + else: + service_name = inspect_result.get('name', 'N/A') + if not isinstance(remote_methods, dict): + (longest_name_len, + detailled_methods) = zerorpc_inspect_python_argspecs( + remote_methods, method, long_doc, include_argspec) - return zerorpc_inspect_generic(remote_methods, method, long_doc, - include_argspec) + (longest_name_len, detailled_methods) = zerorpc_inspect_generic( + remote_methods, method, long_doc, include_argspec) + + return longest_name_len, detailled_methods, service_name def run_client(args): @@ -218,8 +237,9 @@ def run_client(args): client.debug = True setup_links(args, client) if not args.command: - (longest_name_len, detailled_methods) = zerorpc_inspect(client, + (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, long_doc=False, include_argspec=args.inspect) + print '[{0}]'.format(service) if args.inspect: for (name, doc) in detailled_methods: print name @@ -228,10 +248,13 @@ def run_client(args): print '{0} {1}'.format(name.ljust(longest_name_len), doc) return if args.inspect: - (longest_name_len, detailled_methods) = zerorpc_inspect(client, + (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, method=args.command) - (name, doc) = detailled_methods[0] - print '\n{0}\n\n{1}\n'.format(name, doc) + if detailled_methods: + (name, doc) = detailled_methods[0] + print '[{0}]\n{1}\n\n{2}\n'.format(service, name, doc) + else: + print '[{0}]\nNo documentation for "{1}".'.format(service, args.command) return if args.json: call_args = [json.loads(x) for x in args.params] From fec4d82bd5e646935a0b4a8be70d62aceef16ae7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Mon, 3 Mar 2014 01:46:04 -0800 Subject: [PATCH 058/113] Allow exporting non-hashable function Instead of hashing the function object, hash its name instead. There is no particular reason to consider a function instance and its name as two independent entities in the context of the same object. As a bonus, this means you can now replace your methods on the fly and the next call will reach the new one. I am not sure why you would want to do that though. --- zerorpc/core.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/zerorpc/core.py b/zerorpc/core.py index 6758cff..de4e625 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -68,16 +68,15 @@ def __init__(self, channel, methods=None, name=None, context=None, @staticmethod def _filter_methods(cls, self, methods): - if hasattr(methods, '__getitem__'): + if isinstance(methods, dict): return methods - server_methods = set(getattr(self, k) for k in dir(cls) if not - k.startswith('_')) + server_methods = set(k for k in dir(cls) if not k.startswith('_')) return dict((k, getattr(methods, k)) - for k in dir(methods) - if (callable(getattr(methods, k)) - and not k.startswith('_') - and getattr(methods, k) not in server_methods - )) + for k in dir(methods) + if callable(getattr(methods, k)) + and not k.startswith('_') + and k not in server_methods + ) @staticmethod def _extract_name(methods): From a79a0a50318ddb96b83762d1fb924af4ab9eb46b Mon Sep 17 00:00:00 2001 From: Dan Rowles Date: Mon, 5 Jan 2015 14:34:24 +0000 Subject: [PATCH 059/113] If an on_close_if handler in a BufferedChannel decides to close the channel, don't try to send any more _zpc_more messages, as they can no longer be sent. --- tests/test_buffered_channel.py | 58 ++++++++++++++++++++++++++++++++++ zerorpc/channel.py | 5 ++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index f55993c..3c8855e 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -444,3 +444,61 @@ def _do_with_assert_raises(): client.close() server_bufchan.close() server.close() + + +def test_on_close_if(): + """ + Test that the on_close_if method does not cause exceptions when the client + is slow to recv() data. + """ + endpoint = random_ipc_endpoint() + server_events = zerorpc.Events(zmq.ROUTER) + server_events.bind(endpoint) + server = zerorpc.ChannelMultiplexer(server_events) + + client_events = zerorpc.Events(zmq.DEALER) + client_events.connect(endpoint) + client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) + + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=10) + + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=10) + + seen = [] + + def is_stream_done(event): + return event.name == 'done' + + def client_do(): + while True: + event = client_bufchan.recv() + if event.name == 'done': + return + seen.append(event.args) + gevent.sleep(0.1) + + def server_do(): + for i in range(0, 10): + server_bufchan.emit('blah', (i)) + server_bufchan.emit('done', ('bye')) + + client_bufchan.on_close_if = is_stream_done + + coro_pool = gevent.pool.Pool() + g1 = coro_pool.spawn(client_do) + g2 = coro_pool.spawn(server_do) + + g1.get() # Re-raise any exceptions... + g2.get() + + assert seen == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + client_bufchan.close() + server_bufchan.close() + client.close() + server.close() diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 5da8991..1974370 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -254,7 +254,10 @@ def _request_data(self): self._channel.emit('_zpc_more', (open_slots,)) def recv(self, timeout=None): - if self._verbose: + # self._channel can be set to None by an 'on_close_if' callback if it + # sees a suitable message from the remote end... + # + if self._verbose and self._channel: if self._input_queue_reserved < self._input_queue_size / 2: self._request_data() else: From 3ba8bb5510ecffd6ac33251285da8bd260de468b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Mon, 18 May 2015 18:40:18 -0700 Subject: [PATCH 060/113] (c) change. AUTHOR was removed, since commits are better proof of authorship. --- AUTHORS | 6 -- LICENSE | 2 +- README.rst | 4 +- bin/zerorpc | 2 +- doc/protocol.md | 74 +++++++++++++++------- setup.py | 4 +- tests/test_buffered_channel.py | 2 +- tests/test_channel.py | 2 +- tests/test_client.py | 2 +- tests/test_client_async.py | 2 +- tests/test_client_heartbeat.py | 2 +- tests/test_events.py | 2 +- tests/test_heartbeat.py | 2 +- tests/test_middleware.py | 2 +- tests/test_middleware_before_after_exec.py | 2 +- tests/test_middleware_client.py | 2 +- tests/test_pubpush.py | 2 +- tests/test_reqstream.py | 2 +- tests/test_server.py | 2 +- tests/test_wrapped_events.py | 2 +- tests/test_zmq.py | 2 +- tests/testutils.py | 2 +- tests/zmqbug.py | 2 +- zerorpc/__init__.py | 2 +- zerorpc/channel.py | 2 +- zerorpc/cli.py | 2 +- zerorpc/context.py | 2 +- zerorpc/core.py | 2 +- zerorpc/decorators.py | 2 +- zerorpc/events.py | 2 +- zerorpc/exceptions.py | 2 +- zerorpc/gevent_zmq.py | 2 +- zerorpc/heartbeat.py | 2 +- zerorpc/patterns.py | 2 +- zerorpc/socket.py | 2 +- zerorpc/version.py | 6 +- 36 files changed, 88 insertions(+), 68 deletions(-) delete mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 6e329ee..0000000 --- a/AUTHORS +++ /dev/null @@ -1,6 +0,0 @@ -Andrea Luzzardi -François-Xavier Bourlet -Jérôme Petazzoni -Samuel Alba -Solomon Hykes -Sébastien Pahl diff --git a/LICENSE b/LICENSE index 43fb7a1..49bbb6e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ Open Source Initiative OSI - The MIT License (MIT):Licensing The MIT License (MIT) -Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.rst b/README.rst index 21eba59..1210822 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ zerorpc ======= -.. image:: https://secure.travis-ci.org/dotcloud/zerorpc-python.png - :target: http://travis-ci.org/dotcloud/zerorpc-python +.. image:: https://secure.travis-ci.org/0rpc/zerorpc-python.png + :target: http://travis-ci.org/0rpc/zerorpc-python Mailing list: zerorpc@googlegroups.com (https://groups.google.com/d/forum/zerorpc) diff --git a/bin/zerorpc b/bin/zerorpc index 41ab2dc..f0e1172 100755 --- a/bin/zerorpc +++ b/bin/zerorpc @@ -3,7 +3,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/doc/protocol.md b/doc/protocol.md index 2dc3bc0..a5ee1ff 100644 --- a/doc/protocol.md +++ b/doc/protocol.md @@ -1,25 +1,51 @@ -# ZeroRPC Protocol +# zerorpc Protocol THIS DOCUMENT IS INCOMPLETE, WORK IN PROGRESS! -This document attempts to define the ZeroRPC protocol. +This document attempts to define the zerorpc protocol. ## Introduction & History -[ZeroRPC](http://zerorpc.dotcloud.com) is a modern communication layer for distributed systems built on top of [ZeroMQ](http://zeromq.org), developed at [dotCloud](http://dotcloud.com) since 2010 and open-sourced in 2012. It features a dead-simple API for exposing any object or module over the network, and a powerful gevent implementation which supports multiple ZMQ socket types, streaming, heartbeats and more. It also includes a simple command-line tool for interactive querying and introspection. The platform team at dotCloud uses it in production to transmit millions of messages per day across hundreds of services. +zerorpc features a dead-simple API for exposing any object or module over the +network, and a powerful gevent implementation which supports multiple ZMQ +socket types, streaming, heartbeats and more. It also includes a simple +command-line tool for interactive querying and introspection. -ZeroRPC uses ZMQ as a transport, but uses a communication protocol that is transport-agnostic. For a long time the reference documentation for that protocol was the python code itself. However since its recent surge in popularity many people have expressed interest in porting it to other programming languages. We hope that this standalone protocol documentation will help them. - -A short warning: ZeroRPC started as a simple tool to solve a simple problem. It was progressively refined and improved to satisfy the growing needs of the dotCloud platform. The emphasis is on practicality, robustness and operational simplicity - sometimes at the expense of formal elegance and optimizations. We will gladly welcome patches focused on the latter so long as they don't hurt the former. +zerorpc uses ZMQ as a transport, but uses a communication protocol that is +transport-agnostic. For a long time the reference documentation for that +protocol was the python code itself. However since its recent surge in +popularity many people have expressed interest in porting it to other +programming languages. We hope that this standalone protocol documentation will +help them. > The python implementation of zerorpc act as a reference for the whole > protocol. New features and experiments are implemented and tested in this -> version first. This is also this implementation that is powering dotCloud's -> infrastructure. +> version first. + +[zerorpc](http://www.zerorpc.io) is a modern communication layer for +distributed systems built on top of [ZeroMQ](http://zeromq.org), initially +developed at [dotCloud](http://www.dotcloud.com) starting in 2010 and +open-sourced in 2012. when dotCloud pivoted to [Docker](http://www.docker.com), +dotCloud was acquired by [cloudControl](https://www.cloudcontrol.com/), which +then migrated over to theirs own PaaS before shutting it down. + +In 2015, I (François-Xavier Bourlet) was given zerorpc by cloudControl, in an +attempt to revive the project, maintain it, and hopefully drive its +development. + +### Warning + +A short warning: zerorpc started as a simple tool to solve a simple problem. It +was progressively refined and improved to satisfy the growing needs of the +dotCloud platform. The emphasis is on practicality, robustness and operational +simplicity - sometimes at the expense of formal elegance and optimizations. We +will gladly welcome patches focused on the latter so long as they don't hurt +the former. + ## Layers -Before diving into any details, let's divide ZeroRPC's protocol in three +Before diving into any details, let's divide zerorpc's protocol in three different layers: 1. Wire (or transport) layer; a combination of ZMQ @@ -34,15 +60,15 @@ The wire layer is a combination of ZMQ and msgpack. The basics: - - A ZeroRPC server can listen on as many ZMQ sockets as you like. Actually, + - A zerorpc server can listen on as many ZMQ sockets as you like. Actually, a ZMQ socket can bind to multiple addresses. It can even *connect* to the clients (think about it as a worker connecting to a hub), but there are - some limitations in that case (see below). ZeroRPC doesn't + some limitations in that case (see below). zerorpc doesn't have to do anything specific for that: ZMQ handles it automatically. - - A ZeroRPC client can connect to multiple ZeroRPC servers. However, it should + - A zerorpc client can connect to multiple zerorpc servers. However, it should create a new ZMQ socket for each connection. -Since ZeroRPC implements heartbeat and streaming, it expects a kind of +Since zerorpc implements heartbeat and streaming, it expects a kind of persistent, end-to-end, connection between the client and the server. It means that we cannot use the load-balancing features built into ZMQ. Otherwise, the various messages composing a single conversation could @@ -52,12 +78,12 @@ That's why there are limitations when the server connects to the client: if there are multiple servers connecting to the same client, bad things will happen. -> Note that the current implementation of ZeroRPC for Python doesn't implement +> Note that the current implementation of zerorpc for Python doesn't implement > its own load-balancing (yet), and still uses one ZMQ socket for connecting to > many servers. You can still use ZMQ load-balancing if you accept to disable > heartbeat and don't use streamed responses. -Every event from the event layer will be serialized with msgpack. +Every event from the event layer will be serialized with msgpack. ## Event layer @@ -140,18 +166,18 @@ size of its local buffer. This is a hint for the remote, to tell it "send me more data!" - Event's name: '\_zpc\_more' - - Event's args: integer representing how many entries are available in the client's buffer. + - Event's args: integer representing how many entries are available in the client's buffer. FIXME WIP ## RPC Layer -In the first version of ZeroRPC, this was the main (and only) layer. +In the first version of zerorpc, this was the main (and only) layer. Three kinds of events can occur at this layer: request (=function call), -response (=function return), error (=exception). +response (=function return), error (=exception). Request: - + - Event's name: string with the name of the method to call. - Event's args: tuple of arguments for the method. @@ -160,14 +186,14 @@ support them. If you absolutely want to call functions with keyword arguments, you can use a wrapper; e.g. expose a function like "call_with_kwargs(function_name, args, kwargs)", where args is a list, and kwargs a dict. It might be an interesting idea to add such a -helper function into ZeroRPC default methods (see below for definitions +helper function into zerorpc default methods (see below for definitions of existing default methods). Response: - Event's name: string "OK" - Event's args: tuple containing the returned value - + > Note that if the return value is a tuple, it is itself wrapped inside a > tuple - again, for backward compatibility reasons. @@ -178,7 +204,7 @@ exception is raised), we use the ERR event. - Event's name: string "ERR" - Event's args: tuple of 3 strings: - - Name of the error (it should be the exception class name, or another + - Name of the error (it should be the exception class name, or another meaningful keyword). - Human representation of the error (preferably in english). - If possible a pretty printed traceback of the call stack when the error occured. @@ -190,7 +216,7 @@ exception is raised), we use the ERR event. ### Default calls -When exposing some code with ZeroRPC, a number of methods/functions are +When exposing some code with zerorpc, a number of methods/functions are automatically added, to provide useful debugging and development tools. - \_zerorpc\_ping() just answers with a pong message. @@ -213,7 +239,7 @@ Messages part of a stream: - Event's name: string "STREAM" - Event's args: tuple containing the streamed value - + When the STREAM reaches its end: - Event's name: string "STREAM\_DONE" diff --git a/setup.py b/setup.py index 13d4149..829bb02 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -48,7 +48,7 @@ version=__version__, description='zerorpc is a flexible RPC based on zeromq.', author=__author__, - url='https://github.com/dotcloud/zerorpc-python', + url='https://github.com/0rpc/zerorpc-python', packages=['zerorpc'], install_requires=requirements, tests_require=['nose'], diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index f55993c..2d37b9e 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_channel.py b/tests/test_channel.py index ed8367e..4e8ceb7 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_client.py b/tests/test_client.py index 599b0e1..7a954ba 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_client_async.py b/tests/test_client_async.py index 76e0abe..28e0594 100644 --- a/tests/test_client_async.py +++ b/tests/test_client_async.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2013 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_client_heartbeat.py b/tests/test_client_heartbeat.py index e9de869..e276fa7 100644 --- a/tests/test_client_heartbeat.py +++ b/tests/test_client_heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_events.py b/tests/test_events.py index 19ca208..ac2c2bb 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index a4489e1..d37fa67 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3d55b0f..02efe8a 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_middleware_before_after_exec.py b/tests/test_middleware_before_after_exec.py index 23c2e92..40c6578 100644 --- a/tests/test_middleware_before_after_exec.py +++ b/tests/test_middleware_before_after_exec.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_middleware_client.py b/tests/test_middleware_client.py index 0315100..0236620 100644 --- a/tests/test_middleware_client.py +++ b/tests/test_middleware_client.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_pubpush.py b/tests/test_pubpush.py index 7975cee..ac93711 100644 --- a/tests/test_pubpush.py +++ b/tests/test_pubpush.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index ee30423..fc00451 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_server.py b/tests/test_server.py index 8ac48f0..7f25f66 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py index 43c42fe..30a6223 100644 --- a/tests/test_wrapped_events.py +++ b/tests/test_wrapped_events.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/test_zmq.py b/tests/test_zmq.py index 6e9dad1..754e3d5 100644 --- a/tests/test_zmq.py +++ b/tests/test_zmq.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/testutils.py b/tests/testutils.py index 175967c..c6a10f3 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/tests/zmqbug.py b/tests/zmqbug.py index 83b335b..3544aab 100644 --- a/tests/zmqbug.py +++ b/tests/zmqbug.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/__init__.py b/zerorpc/__init__.py index 505ba3c..23e6894 100644 --- a/zerorpc/__init__.py +++ b/zerorpc/__init__.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 5da8991..c997f59 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/cli.py b/zerorpc/cli.py index b7e6135..52432b8 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -3,7 +3,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/context.py b/zerorpc/context.py index 879e6c4..f49a216 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/core.py b/zerorpc/core.py index f30d076..97b133a 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/decorators.py b/zerorpc/decorators.py index f662e7e..60cf1f8 100644 --- a/zerorpc/decorators.py +++ b/zerorpc/decorators.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/events.py b/zerorpc/events.py index c358951..e06a220 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/exceptions.py b/zerorpc/exceptions.py index 781200e..14a1419 100644 --- a/zerorpc/exceptions.py +++ b/zerorpc/exceptions.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index a5cd67e..b4c89e3 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index c4e7c4b..ce005f4 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index 526356c..be8c75d 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/socket.py b/zerorpc/socket.py index 2a7020e..e25a881 100644 --- a/zerorpc/socket.py +++ b/zerorpc/socket.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/version.py b/zerorpc/version.py index 76a6ecf..27dac47 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -24,6 +24,6 @@ __title__ = 'zerorpc' __version__ = '0.4.4' -__author__ = 'dotCloud, Inc.' +__author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' -__copyright__ = 'Copyright 2012 dotCloud, Inc.' +__copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From 511b155758c5285db1d8180f48903f38ca65584f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 26 May 2015 02:13:10 -0700 Subject: [PATCH 061/113] Fix little flake8' complain --- zerorpc/core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zerorpc/core.py b/zerorpc/core.py index 97f6dfc..0bd820c 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -72,11 +72,11 @@ def _filter_methods(cls, self, methods): return methods server_methods = set(k for k in dir(cls) if not k.startswith('_')) return dict((k, getattr(methods, k)) - for k in dir(methods) - if callable(getattr(methods, k)) - and not k.startswith('_') - and k not in server_methods - ) + for k in dir(methods) + if callable(getattr(methods, k)) + and not k.startswith('_') + and k not in server_methods + ) @staticmethod def _extract_name(methods): From fcb6176a3fee7e08447e48a2e12dd93c2bcbf27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 16 Jun 2015 02:21:11 -0700 Subject: [PATCH 062/113] bump version to v0.5.0 --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index 27dac47..445b2bd 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.4.4' +__version__ = '0.5.0' __author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' __copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From 6d04e0d5948c53ec6b3fde21af01242f5ea97575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 16 Jun 2015 02:54:02 -0700 Subject: [PATCH 063/113] bump version to v0.5.1 --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index 445b2bd..3208b41 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.5.0' +__version__ = '0.5.1' __author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' __copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From a83c120fc9821def288bf878e7567df5ed18c299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 11 Sep 2015 18:38:55 -0700 Subject: [PATCH 064/113] Python 2.6 compat: get the PyZmq Frame Buffer A pyzmq frame in python2.6 and 2.7 are not similar. They both offer the attribute buffer which is: - a read-only buffer in 2.6 - a memoryview in 2.7 Apparently the read-only buffer doesn't implement the buffer interface while the memoryview does, which is needed for msgpack. Both accept to be sliced though which returns: - an 'str' in 2.6 - a memoryview in 2.7 Sadly, until very recently, taking a slice of a memoryview of dimension raises an exception: https://bugs.python.org/issue15944. Although it looks like recent version of pyzqm>14.2 are not affected. Anyway, the easiest is access the buffer differently depending of the python version. Fixes 0rpc/zerorpc-python#125. Additionally fixes the unit tests for 2.6 and add it back to travis. --- .travis.yml | 1 + tests/test_buffered_channel.py | 9 ++++----- zerorpc/events.py | 11 ++++++++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4f2163..dec8728 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: python python: + - 2.6 - 2.7 env: matrix: diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index adb27d6..23b7c70 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -166,7 +166,7 @@ def server_fn(): client_bufchan.close() client.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, server_coro.get()) + assert_raises(zerorpc.LostRemote, server_coro.get) else: with assert_raises(zerorpc.LostRemote): server_coro.get() @@ -393,7 +393,7 @@ def test_congestion_control_server_pushing(): client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - read_cnt = 0 + read_cnt = type('Dummy', (object,), { "value": 0 }) def client_do(): client_channel = client.channel() @@ -403,8 +403,7 @@ def client_do(): event = client_bufchan.recv() assert event.name == 'coucou' assert event.args == x - global read_cnt - read_cnt += 1 + read_cnt.value += 1 client_bufchan.close() coro_pool = gevent.pool.Pool() @@ -434,7 +433,7 @@ def _do_with_assert_raises(): with assert_raises(zerorpc.TimeoutExpired): for x in xrange(2, 200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 - for x in xrange(read_cnt, 200): + for x in xrange(read_cnt.value, 200): server_bufchan.emit('coucou', x) # block until receiver is ready server_bufchan.close() diff --git a/zerorpc/events.py b/zerorpc/events.py index ae82fb1..f009505 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -30,6 +30,7 @@ import gevent.local import gevent.lock import logging +import sys import gevent_zmq as zmq from .exceptions import TimeoutExpired @@ -37,6 +38,14 @@ from .channel_base import ChannelBase +if sys.version_info < (2, 7): + def get_pyzmq_frame_buffer(frame): + return frame.buffer[:] +else: + def get_pyzmq_frame_buffer(frame): + return frame.buffer + + class SequentialSender(object): def __init__(self, socket): @@ -329,7 +338,7 @@ def recv(self, timeout=None): else: identity = None blob = parts[0] - event = Event.unpack(blob) + event = Event.unpack(get_pyzmq_frame_buffer(blob)) event.identity = identity if self._debug: logging.debug('<-- %s', event) From 75df686a2c0d94fd164bb796708a866d2b43ddd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 11 Sep 2015 19:04:26 -0700 Subject: [PATCH 065/113] Switch to travis' new infrastructure. + update pyzmq matrix --- .travis.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index dec8728..84bace9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - 2.7 env: matrix: + - PYZMQ='pyzmq>=14' - PYZMQ='pyzmq>=14.3' - PYZMQ='pyzmq>=14.2,<14.3' - PYZMQ='pyzmq>=14.1,<14.2' @@ -11,12 +12,15 @@ env: - PYZMQ='pyzmq<14' matrix: fast_finish: true -script: +script: - flake8 --ignore=E501,E128 zerorpc bin - - ZPC_TEST_TIME_FACTOR=0.1 nosetests -before_install: - - sudo apt-get update - - sudo apt-get install python-dev libevent-dev + - ZPC_TEST_TIME_FACTOR=0.2 nosetests -v +sudo: false +addons: + apt: + packages: + - python-dev + - libevent-dev install: - pip install flake8 - "pip install nose gevent msgpack-python $PYZMQ" From 6a583808c08590f5fd0e21bf5aa5b66d54df5708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 11 Sep 2015 22:42:26 -0700 Subject: [PATCH 066/113] bump version to v0.5.2 --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index 3208b41..00742c9 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.5.1' +__version__ = '0.5.2' __author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' __copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From da8815cbd8209961783c41b15ce44e9bddf0d267 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 17 Sep 2015 14:13:18 +0200 Subject: [PATCH 067/113] Fix loggers names. --- zerorpc/channel.py | 9 ++++----- zerorpc/events.py | 15 +++++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 7a43f7a..59ecb15 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -32,9 +32,8 @@ from .exceptions import TimeoutExpired from .channel_base import ChannelBase -from logging import getLogger -logger = getLogger(__name__) +logger = logging.getLogger(__name__) class ChannelMultiplexer(ChannelBase): @@ -123,7 +122,7 @@ def __init__(self, multiplexer, from_event=None): self._channel_id = from_event.header['message_id'] self._zmqid = from_event.identity self._multiplexer._active_channels[self._channel_id] = self - logging.debug('<-- new channel %s', self._channel_id) + logger.debug('<-- new channel %s', self._channel_id) self._queue.put(from_event) @property @@ -137,7 +136,7 @@ def emit_is_supported(self): def close(self): if self._channel_id is not None: del self._multiplexer._active_channels[self._channel_id] - logging.debug('-x- closed channel %s', self._channel_id) + logger.debug('-x- closed channel %s', self._channel_id) self._channel_id = None def new_event(self, name, args, xheader=None): @@ -145,7 +144,7 @@ def new_event(self, name, args, xheader=None): if self._channel_id is None: self._channel_id = event.header['message_id'] self._multiplexer._active_channels[self._channel_id] = self - logging.debug('--> new channel %s', self._channel_id) + logger.debug('--> new channel %s', self._channel_id) else: event.header['response_to'] = self._channel_id event.identity = self._zmqid diff --git a/zerorpc/events.py b/zerorpc/events.py index f009505..8bc9723 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -46,6 +46,9 @@ def get_pyzmq_frame_buffer(frame): return frame.buffer +logger = logging.getLogger(__name__) + + class SequentialSender(object): def __init__(self, socket): @@ -281,9 +284,9 @@ def debug(self, v): if v != self._debug: self._debug = v if self._debug: - logging.debug('debug enabled') + logger.debug('debug enabled') else: - logging.debug('debug disabled') + logger.debug('debug disabled') def _resolve_endpoint(self, endpoint, resolve=True): if resolve: @@ -299,14 +302,14 @@ def connect(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.connect(endpoint_)) - logging.debug('connected to %s (status=%s)', endpoint_, r[-1]) + logger.debug('connected to %s (status=%s)', endpoint_, r[-1]) return r def bind(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.bind(endpoint_)) - logging.debug('bound to %s (status=%s)', endpoint_, r[-1]) + logger.debug('bound to %s (status=%s)', endpoint_, r[-1]) return r def new_event(self, name, args, xheader=None): @@ -317,7 +320,7 @@ def new_event(self, name, args, xheader=None): def emit_event(self, event, timeout=None): if self._debug: - logging.debug('--> %s', event) + logger.debug('--> %s', event) if event.identity: parts = list(event.identity or list()) parts.extend(['', event.pack()]) @@ -341,7 +344,7 @@ def recv(self, timeout=None): event = Event.unpack(get_pyzmq_frame_buffer(blob)) event.identity = identity if self._debug: - logging.debug('<-- %s', event) + logger.debug('<-- %s', event) return event def setsockopt(self, *args): From 6ca6fdf598905ca67f2678973034dae435a28594 Mon Sep 17 00:00:00 2001 From: Wang Yanqing Date: Sat, 26 Sep 2015 22:55:04 +0800 Subject: [PATCH 068/113] Update msgpack protocol to msgpack 2.0(msgpack-python>=0.4.0) Add use_bin_type to msgpack Packer and set utf-8 encoding for msgpack Unpacker. --- setup.py | 2 +- zerorpc/events.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 1a22102..80846d5 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ requirements = [ 'gevent>=1.0', - 'msgpack-python', + 'msgpack-python>=0.4.0', 'pyzmq>=13.1.0' ] if sys.version_info < (2, 7): diff --git a/zerorpc/events.py b/zerorpc/events.py index f009505..44a2fdb 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -189,11 +189,11 @@ def identity(self, v): self._identity = v def pack(self): - return msgpack.Packer().pack((self._header, self._name, self._args)) + return msgpack.Packer(use_bin_type=True).pack((self._header, self._name, self._args)) @staticmethod def unpack(blob): - unpacker = msgpack.Unpacker() + unpacker = msgpack.Unpacker(encoding='utf-8') unpacker.feed(blob) unpacked_msg = unpacker.unpack() From ab69b94e6a865a5f31291dc59719c69e85ec1348 Mon Sep 17 00:00:00 2001 From: Wang Yanqing Date: Sat, 26 Sep 2015 23:25:28 +0800 Subject: [PATCH 069/113] add unittest for msgpack string type --- tests/test_events.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_events.py b/tests/test_events.py index e3d1616..224a41c 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -169,3 +169,59 @@ def test_events_push_pull(): print event assert event.name == 'myevent' assert list(event.args) == [x] + + +def test_msgpack(): + context = zerorpc.Context() + event = zerorpc.Event('myevent', ('a',), context=context) + print event + assert type(event.name) == str + for key in event.header.keys(): + assert type(key) == str + assert type(event.header['message_id']) == str + assert type(event.args[0]) == str + + packed = event.pack() + event = event.unpack(packed) + print event + assert type(event.name) == str + for key in event.header.keys(): + assert type(key) == str + assert type(event.header['message_id']) == str + assert type(event.args[0]) == str + + event = zerorpc.Event('myevent', (u'a',), context=context) + print event + assert type(event.name) == str + for key in event.header.keys(): + assert type(key) == str + assert type(event.header['message_id']) == str + assert type(event.args[0]) == unicode + + packed = event.pack() + event = event.unpack(packed) + print event + assert type(event.name) == str + for key in event.header.keys(): + assert type(key) == str + assert type(event.header['message_id']) == str + assert type(event.args[0]) == unicode + + event = zerorpc.Event('myevent', (u'a', 'b'), context=context) + print event + assert type(event.name) == str + for key in event.header.keys(): + assert type(key) == str + assert type(event.header['message_id']) == str + assert type(event.args[0]) == unicode + assert type(event.args[1]) == str + + packed = event.pack() + event = event.unpack(packed) + print event + assert type(event.name) == str + for key in event.header.keys(): + assert type(key) == str + assert type(event.header['message_id']) == str + assert type(event.args[0]) == unicode + assert type(event.args[1]) == str From 7fffc42321aa2c25f922d58b55c0fb8463fe0ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Nowak?= Date: Wed, 24 Feb 2016 17:41:42 +0100 Subject: [PATCH 070/113] Added TypeError to events.Events __del__ On interpreter shutdown you may hit TypeError("'NoneType' object is not callable",) --- zerorpc/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/events.py b/zerorpc/events.py index 615426a..9c95709 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -261,7 +261,7 @@ def __del__(self): try: if not self._socket.closed: self.close() - except AttributeError: + except (AttributeError, TypeError): pass def close(self): From e36bcccee65813f6291045ae0b764665c178853a Mon Sep 17 00:00:00 2001 From: Wang Yanqing Date: Thu, 3 Mar 2016 14:57:41 +0800 Subject: [PATCH 071/113] Add disconnct option for zrpc socket --- zerorpc/events.py | 7 +++++++ zerorpc/socket.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/zerorpc/events.py b/zerorpc/events.py index 44a2fdb..7c289e3 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -309,6 +309,13 @@ def bind(self, endpoint, resolve=True): logging.debug('bound to %s (status=%s)', endpoint_, r[-1]) return r + def disconnect(self, endpoint, resolve=True): + r = [] + for endpoint_ in self._resolve_endpoint(endpoint, resolve): + r.append(self._socket.disconnect(endpoint_)) + logging.debug('disconnected from %s (status=%s)', endpoint_, r[-1]) + return r + def new_event(self, name, args, xheader=None): event = Event(name, args, context=self._context) if xheader: diff --git a/zerorpc/socket.py b/zerorpc/socket.py index 51f99c2..35cb7e4 100644 --- a/zerorpc/socket.py +++ b/zerorpc/socket.py @@ -42,6 +42,9 @@ def connect(self, endpoint, resolve=True): def bind(self, endpoint, resolve=True): return self._events.bind(endpoint, resolve) + def disconnect(self, endpoint, resolve=True): + return self._events.disconnect(endpoint, resolve) + @property def debug(self): return self._events.debug From 0e98746bd01b8aedf8f12af5e0f898dc9124bb83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 19:33:03 -0700 Subject: [PATCH 072/113] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1210822..e1cd19f 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ zerorpc ======= -.. image:: https://secure.travis-ci.org/0rpc/zerorpc-python.png +.. image:: https://secure.travis-ci.org/0rpc/zerorpc-python.png?branch=master :target: http://travis-ci.org/0rpc/zerorpc-python Mailing list: zerorpc@googlegroups.com (https://groups.google.com/d/forum/zerorpc) From a032789794bb25cacab6c696f9e230dd47ddfb45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 16:29:00 -0700 Subject: [PATCH 073/113] Flake8 configuration in tox.ini Flake8 will read the config from tox.ini by itself. --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 6f68c02..b56bac5 100644 --- a/tox.ini +++ b/tox.ini @@ -8,3 +8,8 @@ deps = commands = flake8 --ignore=E501,E128 zerorpc bin nosetests + +[flake8] +ignore = E501,E128 +filename = *.py,zerorpc +exclude = tests,.git,dist,doc,*.egg-info,__pycache__,setup.py From b02121218b948433924ab624ae2feb2c07db8100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 16:29:23 -0700 Subject: [PATCH 074/113] Few tags for pypi. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 80846d5..8192082 100644 --- a/setup.py +++ b/setup.py @@ -62,5 +62,8 @@ 'Natural Language :: English', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', ), ) From 916f84dcdaa8c3d01b724813cdfe09e6020f2bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 20:54:05 -0700 Subject: [PATCH 075/113] Keepy flake8 happy. --- bin/zerorpc | 2 +- zerorpc/cli.py | 2 +- zerorpc/core.py | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/zerorpc b/bin/zerorpc index f0e1172..c9d31d3 100755 --- a/bin/zerorpc +++ b/bin/zerorpc @@ -27,7 +27,7 @@ import os import sys sys.path.append(os.path.dirname(os.path.dirname(sys.argv[0]))) -from zerorpc import cli +from zerorpc import cli # NOQA if __name__ == "__main__": exit(cli.main()) diff --git a/zerorpc/cli.py b/zerorpc/cli.py index 1766b08..691e861 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -222,7 +222,7 @@ def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): if not isinstance(remote_methods, dict): (longest_name_len, detailled_methods) = zerorpc_inspect_python_argspecs( - remote_methods, method, long_doc, include_argspec) + remote_methods, method, long_doc, include_argspec) (longest_name_len, detailled_methods) = zerorpc_inspect_generic( remote_methods, method, long_doc, include_argspec) diff --git a/zerorpc/core.py b/zerorpc/core.py index 0bd820c..31921d8 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -73,9 +73,8 @@ def _filter_methods(cls, self, methods): server_methods = set(k for k in dir(cls) if not k.startswith('_')) return dict((k, getattr(methods, k)) for k in dir(methods) - if callable(getattr(methods, k)) - and not k.startswith('_') - and k not in server_methods + if callable(getattr(methods, k)) and + not k.startswith('_') and k not in server_methods ) @staticmethod From bd9e6b05a16e414717bb0593391e458fd52d1f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 22:08:01 -0700 Subject: [PATCH 076/113] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e1cd19f..d2b2965 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ zerorpc ======= -.. image:: https://secure.travis-ci.org/0rpc/zerorpc-python.png?branch=master - :target: http://travis-ci.org/0rpc/zerorpc-python +.. image:: https://travis-ci.org/0rpc/zerorpc-python.svg?branch=master + :target: https://travis-ci.org/0rpc/zerorpc-python Mailing list: zerorpc@googlegroups.com (https://groups.google.com/d/forum/zerorpc) From 468659b72de6e95a60fc1dbf14472ffd8e4ca43f Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Tue, 31 May 2016 16:48:26 -0400 Subject: [PATCH 077/113] Fix reference to root logger instead of namespaced logger --- zerorpc/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/events.py b/zerorpc/events.py index 5612ff9..9ab9e67 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -316,7 +316,7 @@ def disconnect(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.disconnect(endpoint_)) - logging.debug('disconnected from %s (status=%s)', endpoint_, r[-1]) + logger.debug('disconnected from %s (status=%s)', endpoint_, r[-1]) return r def new_event(self, name, args, xheader=None): From 6d375b9213ff5cfebdea2be774ab180c1450adcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 28 Feb 2016 20:55:49 -0800 Subject: [PATCH 078/113] Python3 dependencies --- .travis.yml | 1 + setup.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84bace9..fc548af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - 2.6 - 2.7 + - 3.4 env: matrix: - PYZMQ='pyzmq>=14' diff --git a/setup.py b/setup.py index 8192082..b68b42e 100644 --- a/setup.py +++ b/setup.py @@ -35,13 +35,19 @@ requirements = [ - 'gevent>=1.0', 'msgpack-python>=0.4.0', - 'pyzmq>=13.1.0' + 'pyzmq>=13.1.0', + 'future', ] + if sys.version_info < (2, 7): requirements.append('argparse') +if sys.version_info < (3, 0): + requirements.append('gevent>=1.0') +else: + requirements.append('gevent>=1.1rc5') + setup( name='zerorpc', From 72e268dd8c5564e445aa59145b8adfdf3d9f31e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Mon, 29 Feb 2016 00:43:56 -0800 Subject: [PATCH 079/113] Conversion to >=Python3.4 Code is now using the python 3 syntax , but thanks to "future" and "builtins", it is compatible with 2.6 and 2.7. Hopefully everything was done carefully enough to not impair performances on python 2. Note: Code is not functional and tests are not passing yet. A throughout review of str, bytes and unicode strings is still necessary. --- tests/test_buffered_channel.py | 56 ++++++++++--------- tests/test_channel.py | 16 ++++-- tests/test_client.py | 3 +- tests/test_client_async.py | 8 ++- tests/test_client_heartbeat.py | 33 ++++++----- tests/test_events.py | 64 ++++++++++++---------- tests/test_heartbeat.py | 44 ++++++++------- tests/test_middleware.py | 58 +++++++++++--------- tests/test_middleware_before_after_exec.py | 7 ++- tests/test_middleware_client.py | 7 ++- tests/test_pubpush.py | 26 +++++---- tests/test_reqstream.py | 14 +++-- tests/test_server.py | 36 ++++++------ tests/test_zmq.py | 16 +++--- tests/testutils.py | 7 ++- tests/zmqbug.py | 16 +++--- zerorpc/channel.py | 2 +- zerorpc/cli.py | 22 +++++--- zerorpc/context.py | 7 ++- zerorpc/core.py | 15 +++-- zerorpc/events.py | 8 ++- zerorpc/gevent_zmq.py | 6 +- zerorpc/patterns.py | 4 +- 23 files changed, 273 insertions(+), 202 deletions(-) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index 47c6578..76d0fe3 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -23,13 +23,17 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + from nose.tools import assert_raises import gevent import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_bufchan(): @@ -54,14 +58,14 @@ def test_close_server_bufchan(): server_bufchan.recv() gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE SERVER SOCKET!!!' + print('CLOSE SERVER SOCKET!!!') server_bufchan.close() if sys.version_info < (2, 7): assert_raises(zerorpc.LostRemote, client_bufchan.recv) else: with assert_raises(zerorpc.LostRemote): client_bufchan.recv() - print 'CLIENT LOST SERVER :)' + print('CLIENT LOST SERVER :)') client_bufchan.close() server.close() client.close() @@ -89,14 +93,14 @@ def test_close_client_bufchan(): server_bufchan.recv() gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE CLIENT SOCKET!!!' + print('CLOSE CLIENT SOCKET!!!') client_bufchan.close() if sys.version_info < (2, 7): assert_raises(zerorpc.LostRemote, client_bufchan.recv) else: with assert_raises(zerorpc.LostRemote): client_bufchan.recv() - print 'SERVER LOST CLIENT :)' + print('SERVER LOST CLIENT :)') server_bufchan.close() server.close() client.close() @@ -122,14 +126,14 @@ def test_heartbeat_can_open_channel_server_close(): server_bufchan = zerorpc.BufferedChannel(server_hbchan) gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE SERVER SOCKET!!!' + print('CLOSE SERVER SOCKET!!!') server_bufchan.close() if sys.version_info < (2, 7): assert_raises(zerorpc.LostRemote, client_bufchan.recv) else: with assert_raises(zerorpc.LostRemote): client_bufchan.recv() - print 'CLIENT LOST SERVER :)' + print('CLIENT LOST SERVER :)') client_bufchan.close() server.close() client.close() @@ -162,7 +166,7 @@ def server_fn(): server_coro = gevent.spawn(server_fn) gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE CLIENT SOCKET!!!' + print('CLOSE CLIENT SOCKET!!!') client_bufchan.close() client.close() if sys.version_info < (2, 7): @@ -170,7 +174,7 @@ def server_fn(): else: with assert_raises(zerorpc.LostRemote): server_coro.get() - print 'SERVER LOST CLIENT :)' + print('SERVER LOST CLIENT :)') server.close() @@ -189,7 +193,7 @@ def client_do(): client_channel = client.channel() client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - for x in xrange(20): + for x in range(20): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' @@ -205,7 +209,7 @@ def server_do(): server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - for x in xrange(20): + for x in range(20): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) @@ -229,11 +233,11 @@ def test_do_some_req_rep_lost_server(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): - print 'running' + print('running') client_channel = client.channel() client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - for x in xrange(10): + for x in range(10): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' @@ -254,7 +258,7 @@ def server_do(): server_channel = server.channel(event) server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - for x in xrange(10): + for x in range(10): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) @@ -282,7 +286,7 @@ def client_do(): client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - for x in xrange(10): + for x in range(10): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' @@ -298,7 +302,7 @@ def server_do(): server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - for x in xrange(10): + for x in range(10): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) @@ -334,7 +338,7 @@ def client_do(): if sys.version_info < (2, 7): def _do_with_assert_raises(): - for x in xrange(10): + for x in range(10): client_bufchan.emit('sleep', (x,)) event = client_bufchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' @@ -342,7 +346,7 @@ def _do_with_assert_raises(): assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): + for x in range(10): client_bufchan.emit('sleep', (x,)) event = client_bufchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' @@ -360,7 +364,7 @@ def server_do(): if sys.version_info < (2, 7): def _do_with_assert_raises(): - for x in xrange(20): + for x in range(20): event = server_bufchan.recv() assert event.name == 'sleep' gevent.sleep(TIME_FACTOR * event.args[0]) @@ -368,7 +372,7 @@ def _do_with_assert_raises(): assert_raises(zerorpc.LostRemote, _do_with_assert_raises) else: with assert_raises(zerorpc.LostRemote): - for x in xrange(20): + for x in range(20): event = server_bufchan.recv() assert event.name == 'sleep' gevent.sleep(TIME_FACTOR * event.args[0]) @@ -399,7 +403,7 @@ def client_do(): client_channel = client.channel() client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=100) - for x in xrange(200): + for x in range(200): event = client_bufchan.recv() assert event.name == 'coucou' assert event.args == x @@ -416,24 +420,24 @@ def server_do(): server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=100) if sys.version_info < (2, 7): def _do_with_assert_raises(): - for x in xrange(200): + for x in range(200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(200): + for x in range(200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 server_bufchan.emit('coucou', 1) # block until receiver is ready if sys.version_info < (2, 7): def _do_with_assert_raises(): - for x in xrange(2, 200): + for x in range(2, 200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(2, 200): + for x in range(2, 200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 - for x in xrange(read_cnt.value, 200): + for x in range(read_cnt.value, 200): server_bufchan.emit('coucou', x) # block until receiver is ready server_bufchan.close() diff --git a/tests/test_channel.py b/tests/test_channel.py index 18d2d8d..6a85036 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -23,9 +23,13 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_events_channel_client_side(): @@ -42,7 +46,7 @@ def test_events_channel_client_side(): client_channel.emit('someevent', (42,)) event = server.recv() - print event + print(event) assert list(event.args) == [42] assert event.identity is not None @@ -68,16 +72,16 @@ def test_events_channel_client_side_server_send_many(): client_channel.emit('giveme', (10,)) event = server.recv() - print event + print(event) assert list(event.args) == [10] assert event.identity is not None - for x in xrange(10): + for x in range(10): reply_event = server.new_event('someanswer', (x,), xheader=dict(response_to=event.header['message_id'])) reply_event.identity = event.identity server.emit_event(reply_event) - for x in xrange(10): + for x in range(10): event = client_channel.recv() assert list(event.args) == [x] @@ -96,7 +100,7 @@ def test_events_channel_both_side(): client_channel.emit('openthat', (42,)) event = server.recv() - print event + print(event) assert list(event.args) == [42] assert event.name == 'openthat' diff --git a/tests/test_client.py b/tests/test_client.py index 7a954ba..6a692b3 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -23,10 +23,11 @@ # SOFTWARE. +from __future__ import absolute_import import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_client_connect(): endpoint = random_ipc_endpoint() diff --git a/tests/test_client_async.py b/tests/test_client_async.py index 93645fe..3acbd3b 100644 --- a/tests/test_client_async.py +++ b/tests/test_client_async.py @@ -23,13 +23,15 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import from nose.tools import assert_raises import gevent import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_client_server_client_timeout_with_async(): @@ -55,11 +57,11 @@ def add(self, a, b): if sys.version_info < (2, 7): def _do_with_assert_raises(): - print async_result.get() + print(async_result.get()) assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: with assert_raises(zerorpc.TimeoutExpired): - print async_result.get() + print(async_result.get()) client.close() srv.close() diff --git a/tests/test_client_heartbeat.py b/tests/test_client_heartbeat.py index 1d2936a..6b552a4 100644 --- a/tests/test_client_heartbeat.py +++ b/tests/test_client_heartbeat.py @@ -23,10 +23,15 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import next +from builtins import range + import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_client_server_hearbeat(): @@ -48,7 +53,7 @@ def slow(self): client.connect(endpoint) assert client.lolita() == 42 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_server_activate_heartbeat(): @@ -69,7 +74,7 @@ def lolita(self): client.connect(endpoint) assert client.lolita() == 42 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_server_passive_hearbeat(): @@ -93,7 +98,7 @@ def slow(self): client.connect(endpoint) assert client.slow() == 2 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_hb_doesnt_linger_on_streaming(): @@ -103,7 +108,7 @@ class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): - return xrange(42) + return range(42) srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) @@ -112,8 +117,8 @@ def iter(self): client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - assert list(client1.iter()) == list(xrange(42)) - print 'sleep 3s' + assert list(client1.iter()) == list(range(42)) + print('sleep 3s') gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -158,10 +163,10 @@ def iter(self): client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - print 'grab iter' + print('grab iter') i = client1.iter() - print 'sleep 3s' + print('sleep 3s') gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -174,7 +179,7 @@ class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): - return xrange(500) + return range(500) srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) @@ -183,13 +188,13 @@ def iter(self): client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - print 'grab iter' + print('grab iter') i = client1.iter() - print 'consume some' - assert list(next(i) for x in xrange(142)) == list(xrange(142)) + print('consume some') + assert list(next(i) for x in range(142)) == list(range(142)) - print 'sleep 3s' + print('sleep 3s') gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() diff --git a/tests/test_events.py b/tests/test_events.py index 224a41c..91a116b 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -23,11 +23,17 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import str +from builtins import range +from builtins import object + from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint -class MokupContext(): +class MokupContext(object): _next_id = 0 def new_msgid(self): @@ -44,32 +50,32 @@ def test_context(): def test_event(): context = MokupContext() event = zerorpc.Event('mylittleevent', (None,), context=context) - print event + print(event) assert event.name == 'mylittleevent' assert event.header['message_id'] == 0 assert event.args == (None,) event = zerorpc.Event('mylittleevent2', ('42',), context=context) - print event + print(event) assert event.name == 'mylittleevent2' assert event.header['message_id'] == 1 assert event.args == ('42',) event = zerorpc.Event('mylittleevent3', ('a', 42), context=context) - print event + print(event) assert event.name == 'mylittleevent3' assert event.header['message_id'] == 2 assert event.args == ('a', 42) event = zerorpc.Event('mylittleevent4', ('b', 21), context=context) - print event + print(event) assert event.name == 'mylittleevent4' assert event.header['message_id'] == 3 assert event.args == ('b', 21) packed = event.pack() unpacked = zerorpc.Event.unpack(packed) - print unpacked + print(unpacked) assert unpacked.name == 'mylittleevent4' assert unpacked.header['message_id'] == 3 @@ -77,13 +83,13 @@ def test_event(): event = zerorpc.Event('mylittleevent5', ('c', 24, True), header={'lol': 'rofl'}, context=None) - print event + print(event) assert event.name == 'mylittleevent5' assert event.header['lol'] == 'rofl' assert event.args == ('c', 24, True) event = zerorpc.Event('mod', (42,), context=context) - print event + print(event) assert event.name == 'mod' assert event.header['message_id'] == 4 assert event.args == (42,) @@ -102,7 +108,7 @@ def test_events_req_rep(): client.emit('myevent', ('arg1',)) event = server.recv() - print event + print(event) assert event.name == 'myevent' assert list(event.args) == ['arg1'] @@ -115,16 +121,16 @@ def test_events_req_rep2(): client = zerorpc.Events(zmq.REQ) client.connect(endpoint) - for i in xrange(10): + for i in range(10): client.emit('myevent' + str(i), (i,)) event = server.recv() - print event + print(event) assert event.name == 'myevent' + str(i) assert list(event.args) == [i] server.emit('answser' + str(i * 2), (i * 2,)) event = client.recv() - print event + print(event) assert event.name == 'answser' + str(i * 2) assert list(event.args) == [i * 2] @@ -137,10 +143,10 @@ def test_events_dealer_router(): client = zerorpc.Events(zmq.DEALER) client.connect(endpoint) - for i in xrange(6): + for i in range(6): client.emit('myevent' + str(i), (i,)) event = server.recv() - print event + print(event) assert event.name == 'myevent' + str(i) assert list(event.args) == [i] @@ -148,7 +154,7 @@ def test_events_dealer_router(): reply_event.identity = event.identity server.emit_event(reply_event) event = client.recv() - print event + print(event) assert event.name == 'answser' + str(i * 2) assert list(event.args) == [i * 2] @@ -161,12 +167,12 @@ def test_events_push_pull(): client = zerorpc.Events(zmq.PUSH) client.connect(endpoint) - for x in xrange(10): + for x in range(10): client.emit('myevent', (x,)) - for x in xrange(10): + for x in range(10): event = server.recv() - print event + print(event) assert event.name == 'myevent' assert list(event.args) == [x] @@ -174,7 +180,7 @@ def test_events_push_pull(): def test_msgpack(): context = zerorpc.Context() event = zerorpc.Event('myevent', ('a',), context=context) - print event + print(event) assert type(event.name) == str for key in event.header.keys(): assert type(key) == str @@ -183,7 +189,7 @@ def test_msgpack(): packed = event.pack() event = event.unpack(packed) - print event + print(event) assert type(event.name) == str for key in event.header.keys(): assert type(key) == str @@ -191,37 +197,37 @@ def test_msgpack(): assert type(event.args[0]) == str event = zerorpc.Event('myevent', (u'a',), context=context) - print event + print(event) assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str - assert type(event.args[0]) == unicode + assert type(event.args[0]) == str packed = event.pack() event = event.unpack(packed) - print event + print(event) assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str - assert type(event.args[0]) == unicode + assert type(event.args[0]) == str event = zerorpc.Event('myevent', (u'a', 'b'), context=context) - print event + print(event) assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str - assert type(event.args[0]) == unicode + assert type(event.args[0]) == str assert type(event.args[1]) == str packed = event.pack() event = event.unpack(packed) - print event + print(event) assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str - assert type(event.args[0]) == unicode + assert type(event.args[0]) == str assert type(event.args[1]) == str diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 0eee2b5..75b1d29 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -23,13 +23,17 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + from nose.tools import assert_raises import gevent import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_hbchan(): @@ -52,14 +56,14 @@ def test_close_server_hbchan(): server_hbchan.recv() gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE SERVER SOCKET!!!' + print('CLOSE SERVER SOCKET!!!') server_hbchan.close() if sys.version_info < (2, 7): assert_raises(zerorpc.LostRemote, client_hbchan.recv) else: with assert_raises(zerorpc.LostRemote): client_hbchan.recv() - print 'CLIENT LOST SERVER :)' + print('CLIENT LOST SERVER :)') client_hbchan.close() server.close() client.close() @@ -85,14 +89,14 @@ def test_close_client_hbchan(): server_hbchan.recv() gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE CLIENT SOCKET!!!' + print('CLOSE CLIENT SOCKET!!!') client_hbchan.close() if sys.version_info < (2, 7): assert_raises(zerorpc.LostRemote, server_hbchan.recv) else: with assert_raises(zerorpc.LostRemote): server_hbchan.recv() - print 'SERVER LOST CLIENT :)' + print('SERVER LOST CLIENT :)') server_hbchan.close() server.close() client.close() @@ -116,14 +120,14 @@ def test_heartbeat_can_open_channel_server_close(): server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE SERVER SOCKET!!!' + print('CLOSE SERVER SOCKET!!!') server_hbchan.close() if sys.version_info < (2, 7): assert_raises(zerorpc.LostRemote, client_hbchan.recv) else: with assert_raises(zerorpc.LostRemote): client_hbchan.recv() - print 'CLIENT LOST SERVER :)' + print('CLIENT LOST SERVER :)') client_hbchan.close() server.close() client.close() @@ -147,7 +151,7 @@ def test_heartbeat_can_open_channel_client_close(): server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) gevent.sleep(TIME_FACTOR * 3) - print 'CLOSE CLIENT SOCKET!!!' + print('CLOSE CLIENT SOCKET!!!') client_hbchan.close() client.close() if sys.version_info < (2, 7): @@ -155,7 +159,7 @@ def test_heartbeat_can_open_channel_client_close(): else: with assert_raises(zerorpc.LostRemote): server_hbchan.recv() - print 'SERVER LOST CLIENT :)' + print('SERVER LOST CLIENT :)') server_hbchan.close() server.close() @@ -178,7 +182,7 @@ def test_do_some_req_rep(): server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 4) def client_do(): - for x in xrange(20): + for x in range(20): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' @@ -188,7 +192,7 @@ def client_do(): client_task = gevent.spawn(client_do) def server_do(): - for x in xrange(20): + for x in range(20): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) @@ -213,10 +217,10 @@ def test_do_some_req_rep_lost_server(): client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): - print 'running' + print('running') client_channel = client.channel() client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' @@ -235,7 +239,7 @@ def server_do(): event = server.recv() server_channel = server.channel(event) server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) @@ -263,7 +267,7 @@ def client_do(): client_channel = client.channel() client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' @@ -277,7 +281,7 @@ def server_do(): server_channel = server.channel(event) server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) @@ -313,7 +317,7 @@ def client_do(): if sys.version_info < (2, 7): def _do_with_assert_raises(): - for x in xrange(10): + for x in range(10): client_hbchan.emit('sleep', (x,)) event = client_hbchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' @@ -321,7 +325,7 @@ def _do_with_assert_raises(): assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): + for x in range(10): client_hbchan.emit('sleep', (x,)) event = client_hbchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' @@ -337,7 +341,7 @@ def server_do(): if sys.version_info < (2, 7): def _do_with_assert_raises(): - for x in xrange(20): + for x in range(20): event = server_hbchan.recv() assert event.name == 'sleep' gevent.sleep(TIME_FACTOR * event.args[0]) @@ -345,7 +349,7 @@ def _do_with_assert_raises(): assert_raises(zerorpc.LostRemote, _do_with_assert_raises) else: with assert_raises(zerorpc.LostRemote): - for x in xrange(20): + for x in range(20): event = server_hbchan.recv() assert event.name == 'sleep' gevent.sleep(TIME_FACTOR * event.args[0]) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index c97270a..5592713 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -23,6 +23,10 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import str + from nose.tools import assert_raises import gevent import gevent.local @@ -32,7 +36,7 @@ from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_resolve_endpoint(): @@ -47,16 +51,16 @@ def resolve(endpoint): cnt = c.register_middleware({ 'resolve_endpoint': resolve }) - print 'registered_count:', cnt + print('registered_count:', cnt) assert cnt == 1 - print 'resolve titi:', c.hook_resolve_endpoint('titi') + print('resolve titi:', c.hook_resolve_endpoint('titi')) assert c.hook_resolve_endpoint('titi') == test_endpoint - print 'resolve toto:', c.hook_resolve_endpoint('toto') + print('resolve toto:', c.hook_resolve_endpoint('toto')) assert c.hook_resolve_endpoint('toto') == 'toto' - class Resolver(): + class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'toto': @@ -64,18 +68,18 @@ def resolve_endpoint(self, endpoint): return endpoint cnt = c.register_middleware(Resolver()) - print 'registered_count:', cnt + print('registered_count:', cnt) assert cnt == 1 - print 'resolve titi:', c.hook_resolve_endpoint('titi') + print('resolve titi:', c.hook_resolve_endpoint('titi')) assert c.hook_resolve_endpoint('titi') == test_endpoint - print 'resolve toto:', c.hook_resolve_endpoint('toto') + print('resolve toto:', c.hook_resolve_endpoint('toto')) assert c.hook_resolve_endpoint('toto') == test_endpoint c2 = zerorpc.Context() - print 'resolve titi:', c2.hook_resolve_endpoint('titi') + print('resolve titi:', c2.hook_resolve_endpoint('titi')) assert c2.hook_resolve_endpoint('titi') == 'titi' - print 'resolve toto:', c2.hook_resolve_endpoint('toto') + print('resolve toto:', c2.hook_resolve_endpoint('toto')) assert c2.hook_resolve_endpoint('toto') == 'toto' @@ -83,7 +87,7 @@ def test_resolve_endpoint_events(): test_endpoint = random_ipc_endpoint() c = zerorpc.Context() - class Resolver(): + class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'some_service': return test_endpoint @@ -91,7 +95,7 @@ def resolve_endpoint(self, endpoint): class Srv(zerorpc.Server): def hello(self): - print 'heee' + print('heee') return 'world' srv = Srv(heartbeat=TIME_FACTOR * 1, context=c) @@ -114,7 +118,7 @@ def hello(self): srv.close() -class Tracer: +class Tracer(object): '''Used by test_task_context_* tests''' def __init__(self, identity): self._identity = identity @@ -127,7 +131,7 @@ def trace_id(self): def load_task_context(self, event_header): self._locals.trace_id = event_header.get('trace_id', None) - print self._identity, 'load_task_context', self.trace_id + print(self._identity, 'load_task_context', self.trace_id) self._log.append(('load', self.trace_id)) def get_task_context(self): @@ -136,10 +140,10 @@ def get_task_context(self): self._locals.trace_id = '<{0}>'.format(hashlib.md5( str(random.random())[3:] ).hexdigest()[0:6].upper()) - print self._identity, 'get_task_context! [make a new one]', self.trace_id + print(self._identity, 'get_task_context! [make a new one]', self.trace_id) self._log.append(('new', self.trace_id)) else: - print self._identity, 'get_task_context! [reuse]', self.trace_id + print(self._identity, 'get_task_context! [reuse]', self.trace_id) self._log.append(('reuse', self.trace_id)) return { 'trace_id': self.trace_id } @@ -154,7 +158,7 @@ def test_task_context(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -201,7 +205,7 @@ def test_task_context_relay(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -212,7 +216,7 @@ def echo(self, msg): c_relay = zerorpc.Client(context=srv_relay_ctx) c_relay.connect(endpoint1) - class SrvRelay: + class SrvRelay(object): def echo(self, msg): return c_relay.echo('relay' + msg) + 'relayed' @@ -257,7 +261,7 @@ def test_task_context_relay_fork(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -268,15 +272,15 @@ def echo(self, msg): c_relay = zerorpc.Client(context=srv_relay_ctx) c_relay.connect(endpoint1) - class SrvRelay: + class SrvRelay(object): def echo(self, msg): def dothework(msg): return c_relay.echo(msg) + 'relayed' g = gevent.spawn(zerorpc.fork_task_context(dothework, srv_relay_ctx), 'relay' + msg) - print 'relaying in separate task:', g + print('relaying in separate task:', g) r = g.get() - print 'back to main task' + print('back to main task') return r srv_relay = zerorpc.Server(SrvRelay(), context=srv_relay_ctx) @@ -321,7 +325,7 @@ def test_task_context_pushpull(): trigger = gevent.event.Event() - class Puller: + class Puller(object): def echo(self, msg): trigger.set() @@ -359,7 +363,7 @@ def test_task_context_pubsub(): trigger = gevent.event.Event() - class Subscriber: + class Subscriber(object): def echo(self, msg): trigger.set() @@ -381,9 +385,9 @@ def echo(self, msg): subscriber.stop() subscriber_task.join() - print publisher_tracer._log + print(publisher_tracer._log) assert ('new', publisher_tracer.trace_id) in publisher_tracer._log - print subscriber_tracer._log + print(subscriber_tracer._log) assert ('load', publisher_tracer.trace_id) in subscriber_tracer._log diff --git a/tests/test_middleware_before_after_exec.py b/tests/test_middleware_before_after_exec.py index 07821bc..5dafeb0 100644 --- a/tests/test_middleware_before_after_exec.py +++ b/tests/test_middleware_before_after_exec.py @@ -22,10 +22,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import +from builtins import range + import gevent import zerorpc -from testutils import random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR class EchoModule(object): @@ -42,7 +45,7 @@ def echo(self, msg): @zerorpc.stream def echoes(self, msg): self.last_msg = 'echo: ' + msg - for i in xrange(0, 3): + for i in range(0, 3): yield self.last_msg class ServerBeforeExecMiddleware(object): diff --git a/tests/test_middleware_client.py b/tests/test_middleware_client.py index c65eae9..943985e 100644 --- a/tests/test_middleware_client.py +++ b/tests/test_middleware_client.py @@ -22,10 +22,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import +from builtins import range + import gevent import zerorpc -from testutils import random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR class EchoModule(object): @@ -42,7 +45,7 @@ def echo(self, msg): @zerorpc.stream def echoes(self, msg): self.last_msg = "echo: " + msg - for i in xrange(0, 3): + for i in range(0, 3): yield self.last_msg def crash(self, msg): diff --git a/tests/test_pubpush.py b/tests/test_pubpush.py index ac93711..a99f9b4 100644 --- a/tests/test_pubpush.py +++ b/tests/test_pubpush.py @@ -23,11 +23,15 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + import gevent import gevent.event import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_pushpull_inheritance(): @@ -39,7 +43,7 @@ def test_pushpull_inheritance(): class Puller(zerorpc.Puller): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -50,7 +54,7 @@ def lolita(self, a, b): trigger.clear() pusher.lolita(1, 2) trigger.wait() - print 'done' + print('done') def test_pubsub_inheritance(): @@ -62,7 +66,7 @@ def test_pubsub_inheritance(): class Subscriber(zerorpc.Subscriber): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -73,10 +77,10 @@ def lolita(self, a, b): trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). - for attempt in xrange(0, 10): + for attempt in range(0, 10): publisher.lolita(1, 2) if trigger.wait(0.2): - print 'done' + print('done') return raise RuntimeError("The subscriber didn't receive any published message") @@ -87,7 +91,7 @@ def test_pushpull_composite(): class Puller(object): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -102,7 +106,7 @@ def lolita(self, a, b): trigger.clear() pusher.lolita(1, 2) trigger.wait() - print 'done' + print('done') def test_pubsub_composite(): @@ -111,7 +115,7 @@ def test_pubsub_composite(): class Subscriber(object): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -126,10 +130,10 @@ def lolita(self, a, b): trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). - for attempt in xrange(0, 10): + for attempt in range(0, 10): publisher.lolita(1, 2) if trigger.wait(0.2): - print 'done' + print('done') return raise RuntimeError("The subscriber didn't receive any published message") diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index 276d5dd..927019d 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -23,10 +23,14 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_rcp_streaming(): @@ -36,11 +40,11 @@ class MySrv(zerorpc.Server): @zerorpc.rep def range(self, max): - return range(max) + return list(range(max)) @zerorpc.stream def xrange(self, max): - return xrange(max) + return range(max) srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) @@ -55,8 +59,8 @@ def xrange(self, max): r = client.xrange(10) assert getattr(r, 'next', None) is not None l = [] - print 'wait 4s for fun' + print('wait 4s for fun') gevent.sleep(TIME_FACTOR * 4) for x in r: l.append(x) - assert l == range(10) + assert l == list(range(10)) diff --git a/tests/test_server.py b/tests/test_server.py index 548b1e4..200b462 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -23,13 +23,17 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + from nose.tools import assert_raises import gevent import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint, TIME_FACTOR +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_server_manual(): @@ -83,10 +87,10 @@ def add(self, a, b): client = zerorpc.Client() client.connect(endpoint) - print client.lolita() + print(client.lolita()) assert client.lolita() == 42 - print client.add(1, 4) + print(client.add(1, 4)) assert client.add(1, 4) == 5 @@ -113,7 +117,7 @@ def add(self, a, b): assert_raises(zerorpc.TimeoutExpired, client.add, 1, 4) else: with assert_raises(zerorpc.TimeoutExpired): - print client.add(1, 4) + print(client.add(1, 4)) client.close() srv.close() @@ -135,12 +139,12 @@ def raise_something(self, a): if sys.version_info < (2, 7): def _do_with_assert_raises(): - print client.raise_something(42) + print(client.raise_something(42)) assert_raises(zerorpc.RemoteError, _do_with_assert_raises) else: with assert_raises(zerorpc.RemoteError): - print client.raise_something(42) - assert client.raise_something(range(5)) == 4 + print(client.raise_something(42)) + assert client.raise_something(list(range(5))) == 4 client.close() srv.close() @@ -162,17 +166,17 @@ def raise_error(self): if sys.version_info < (2, 7): def _do_with_assert_raises(): - print client.raise_error() + print(client.raise_error()) assert_raises(zerorpc.RemoteError, _do_with_assert_raises) else: with assert_raises(zerorpc.RemoteError): - print client.raise_error() + print(client.raise_error()) try: client.raise_error() except zerorpc.RemoteError as e: - print 'got that:', e - print 'name', e.name - print 'msg', e.msg + print('got that:', e) + print('name', e.name) + print('msg', e.msg) assert e.name == 'RuntimeError' assert e.msg == 'oops!' @@ -197,20 +201,20 @@ class MySrv(zerorpc.Server): rpccall = client.channel() rpccall.emit('donotexist', tuple()) event = rpccall.recv() - print event + print(event) assert event.name == 'ERR' (name, msg, tb) = event.args - print 'detailed error', name, msg, tb + print('detailed error', name, msg, tb) assert name == 'NameError' assert msg == 'donotexist' rpccall = client.channel() rpccall.emit('donotexist', tuple(), xheader=dict(v=1)) event = rpccall.recv() - print event + print(event) assert event.name == 'ERR' (msg,) = event.args - print 'msg only', msg + print('msg only', msg) assert msg == "NameError('donotexist',)" client_events.close() diff --git a/tests/test_zmq.py b/tests/test_zmq.py index 723c941..e3f24d2 100644 --- a/tests/test_zmq.py +++ b/tests/test_zmq.py @@ -23,10 +23,12 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import import gevent from zerorpc import zmq -from testutils import random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test1(): @@ -36,10 +38,10 @@ def server(): s = c.socket(zmq.REP) s.bind(endpoint) while True: - print 'srv recving...' + print('srv recving...') r = s.recv() - print 'srv', r - print 'srv sending...' + print('srv', r) + print('srv sending...') s.send('world') s.close() @@ -50,11 +52,11 @@ def client(): s = c.socket(zmq.REQ) s.connect(endpoint) - print 'cli sending...' + print('cli sending...') s.send('hello') - print 'cli recving...' + print('cli recving...') r = s.recv() - print 'cli', r + print('cli', r) s.close() c.term() diff --git a/tests/testutils.py b/tests/testutils.py index 5b73bc5..eb731ce 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -22,6 +22,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import print_function +from builtins import str + import functools import nose.exc import random @@ -38,7 +41,7 @@ def random_ipc_endpoint(): def teardown(): global _tmpfiles for tmpfile in _tmpfiles: - print 'unlink', tmpfile + print('unlink', tmpfile) try: os.unlink(tmpfile) except Exception: @@ -55,6 +58,6 @@ def wrap(): try: TIME_FACTOR = float(os.environ.get('ZPC_TEST_TIME_FACTOR')) - print 'ZPC_TEST_TIME_FACTOR:', TIME_FACTOR + print('ZPC_TEST_TIME_FACTOR:', TIME_FACTOR) except TypeError: TIME_FACTOR = 1.0 diff --git a/tests/zmqbug.py b/tests/zmqbug.py index 3544aab..1d102a2 100644 --- a/tests/zmqbug.py +++ b/tests/zmqbug.py @@ -25,6 +25,8 @@ # Based on https://github.com/traviscline/gevent-zeromq/blob/master/gevent_zeromq/core.py +from __future__ import print_function + import zmq import gevent.event @@ -79,7 +81,7 @@ def send(self, data, flags=0, copy=True, track=False): while True: try: return super(ZMQSocket, self).send(data, flags, copy, track) - except zmq.ZMQError, e: + except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: raise self._writable.clear() @@ -92,14 +94,14 @@ def recv(self, flags=0, copy=True, track=False): while True: try: return super(ZMQSocket, self).recv(flags, copy, track) - except zmq.ZMQError, e: + except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: raise self._readable.clear() while not self._readable.wait(timeout=10): events = self.getsockopt(zmq.EVENTS) if bool(events & zmq.POLLIN): - print "here we go, nobody told me about new messages!" + print("here we go, nobody told me about new messages!") global STOP_EVERYTHING STOP_EVERYTHING = True raise gevent.GreenletExit() @@ -111,7 +113,7 @@ def server(): socket = ZMQSocket(zmq_context, zmq.REP) socket.bind('ipc://zmqbug') - class Cnt: + class Cnt(object): responded = 0 cnt = Cnt() @@ -125,7 +127,7 @@ def responder(): gevent.spawn(responder) while not STOP_EVERYTHING: - print "cnt.responded=", cnt.responded + print("cnt.responded=", cnt.responded) gevent.sleep(0.5) @@ -133,7 +135,7 @@ def client(): socket = ZMQSocket(zmq_context, zmq.DEALER) socket.connect('ipc://zmqbug') - class Cnt: + class Cnt(object): recv = 0 send = 0 @@ -156,7 +158,7 @@ def sendmsg(): gevent.spawn(sendmsg) while not STOP_EVERYTHING: - print "cnt.recv=", cnt.recv, "cnt.send=", cnt.send + print("cnt.recv=", cnt.recv, "cnt.send=", cnt.send) gevent.sleep(0.5) gevent.spawn(server) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 8c96a5b..e1e6853 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -246,7 +246,7 @@ def recv(self, timeout=None): # sees a suitable message from the remote end... # if self._verbose and self._channel: - if self._input_queue_reserved < self._input_queue_size / 2: + if self._input_queue_reserved < self._input_queue_size // 2: self._request_data() else: self._verbose = True diff --git a/zerorpc/cli.py b/zerorpc/cli.py index 691e861..2985c91 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -24,12 +24,16 @@ # SOFTWARE. +from __future__ import print_function +from builtins import map + import argparse import json import sys import inspect import os import logging +import collections from pprint import pprint import zerorpc @@ -86,7 +90,7 @@ def setup_links(args, socket): if args.bind: for endpoint in args.bind: - print 'binding to "{0}"'.format(endpoint) + print('binding to "{0}"'.format(endpoint)) socket.bind(endpoint) addresses = [] if args.address: @@ -94,7 +98,7 @@ def setup_links(args, socket): if args.connect: addresses.extend(args.connect) for endpoint in addresses: - print 'connecting to "{0}"'.format(endpoint) + print('connecting to "{0}"'.format(endpoint)) socket.connect(endpoint) @@ -116,7 +120,7 @@ def run_server(args): if args.debug: server.debug = True setup_links(args, server) - print 'serving "{0}"'.format(server_obj_path) + print('serving "{0}"'.format(server_obj_path)) return server.run() @@ -239,29 +243,29 @@ def run_client(args): if not args.command: (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, long_doc=False, include_argspec=args.inspect) - print '[{0}]'.format(service) + print('[{0}]'.format(service)) if args.inspect: for (name, doc) in detailled_methods: - print name + print(name) else: for (name, doc) in detailled_methods: - print '{0} {1}'.format(name.ljust(longest_name_len), doc) + print('{0} {1}'.format(name.ljust(longest_name_len), doc)) return if args.inspect: (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, method=args.command) if detailled_methods: (name, doc) = detailled_methods[0] - print '[{0}]\n{1}\n\n{2}\n'.format(service, name, doc) + print('[{0}]\n{1}\n\n{2}\n'.format(service, name, doc)) else: - print '[{0}]\nNo documentation for "{1}".'.format(service, args.command) + print('[{0}]\nNo documentation for "{1}".'.format(service, args.command)) return if args.json: call_args = [json.loads(x) for x in args.params] else: call_args = args.params results = client(args.command, *call_args) - if getattr(results, 'next', None) is None: + if not isinstance(results, collections.Iterator): if args.print_json: json.dump(results, sys.stdout) else: diff --git a/zerorpc/context.py b/zerorpc/context.py index 327f1ff..9600d62 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -23,10 +23,13 @@ # SOFTWARE. +from __future__ import absolute_import +from builtins import str + import uuid import random -import gevent_zmq as zmq +from . import gevent_zmq as zmq class Context(zmq.Context): @@ -114,7 +117,7 @@ def new_msgid(self): def register_middleware(self, middleware_instance): registered_count = 0 self._middlewares.append(middleware_instance) - for hook in self._hooks.keys(): + for hook in self._hooks: functor = getattr(middleware_instance, hook, None) if functor is None: try: diff --git a/zerorpc/core.py b/zerorpc/core.py index 31921d8..72e394e 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -23,6 +23,11 @@ # SOFTWARE. +from __future__ import absolute_import +from builtins import str +from builtins import zip +from future.utils import iteritems + import sys import traceback import gevent.pool @@ -31,14 +36,14 @@ import gevent.local import gevent.lock -import gevent_zmq as zmq +from . import gevent_zmq as zmq from .exceptions import TimeoutExpired, RemoteError, LostRemote from .channel import ChannelMultiplexer, BufferedChannel from .socket import SocketBase from .heartbeat import HeartBeatOnChannel from .context import Context from .decorators import DecoratorBase, rep -import patterns +from . import patterns from logging import getLogger logger = getLogger(__name__) @@ -62,7 +67,7 @@ def __init__(self, channel, methods=None, name=None, context=None, self._inject_builtins() self._heartbeat_freq = heartbeat - for (k, functor) in self._methods.items(): + for (k, functor) in iteritems(self._methods): if not isinstance(functor, DecoratorBase): self._methods[k] = rep(functor) @@ -97,11 +102,11 @@ def _format_args_spec(self, args_spec, r=None): return r def _zerorpc_inspect(self): - methods = dict((m, f) for m, f in self._methods.items() + methods = dict((m, f) for m, f in iteritems(self._methods) if not m.startswith('_')) detailled_methods = dict((m, dict(args=self._format_args_spec(f._zerorpc_args()), - doc=f._zerorpc_doc())) for (m, f) in methods.items()) + doc=f._zerorpc_doc())) for (m, f) in iteritems(methods)) return {'name': self._name, 'methods': detailled_methods} diff --git a/zerorpc/events.py b/zerorpc/events.py index 5612ff9..07f3266 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -23,6 +23,10 @@ # SOFTWARE. +from __future__ import absolute_import +from builtins import str +from builtins import range + import msgpack import gevent.pool import gevent.queue @@ -32,7 +36,7 @@ import logging import sys -import gevent_zmq as zmq +from . import gevent_zmq as zmq from .exceptions import TimeoutExpired from .context import Context from .channel_base import ChannelBase @@ -56,7 +60,7 @@ def __init__(self, socket): def _send(self, parts): e = None - for i in xrange(len(parts) - 1): + for i in range(len(parts) - 1): try: self._socket.send(parts[i], copy=False, flags=zmq.SNDMORE) except (gevent.GreenletExit, gevent.Timeout) as e: diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index b4c89e3..badd8ea 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -100,7 +100,7 @@ def connect(self, *args, **kwargs): while True: try: return super(Socket, self).connect(*args, **kwargs) - except _zmq.ZMQError, e: + except _zmq.ZMQError as e: if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise @@ -122,7 +122,7 @@ def send(self, data, flags=0, copy=True, track=False): # send and recv on the socket. self._on_state_changed() return msg - except _zmq.ZMQError, e: + except _zmq.ZMQError as e: if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._writable.clear() @@ -162,7 +162,7 @@ def recv(self, flags=0, copy=True, track=False): # send and recv on the socket. self._on_state_changed() return msg - except _zmq.ZMQError, e: + except _zmq.ZMQError as e: if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._readable.clear() diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index bf6ede5..8d6ee0a 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -23,7 +23,7 @@ # SOFTWARE. -class ReqRep: +class ReqRep(object): def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) @@ -49,7 +49,7 @@ def process_answer(self, context, channel, req_event, rep_event, channel.close() -class ReqStream: +class ReqStream(object): def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) From f044578b23d403334802134ed8e96a765a3a4f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 19:22:58 -0700 Subject: [PATCH 080/113] Proper handling of bytes vs str (unicode). Something to be careful about as zerorpc is now writtent in python3: - str() must be an unicode string (using builtins.str ensure that on python 2). - bytes() must be a string of bytes (we must use builtins.bytes to get the same behavior on pythong 2). - b"some bytes" is a string of bytes. - u"some unicode" is a string of unicodes. And notably "a string" is a byte string on python 2, but an unicode string on python 3. This means zerorpc between a service and a client using different version of python requires proper testing. --- tests/test_channel.py | 4 +- tests/test_events.py | 86 ++++++++++++++++++++-------------------- tests/test_middleware.py | 3 +- tests/test_reqstream.py | 3 +- tests/test_server.py | 2 +- tests/test_zmq.py | 4 +- zerorpc/channel.py | 8 ++-- zerorpc/context.py | 6 +-- zerorpc/core.py | 6 +-- zerorpc/events.py | 10 +++-- zerorpc/heartbeat.py | 2 +- 11 files changed, 69 insertions(+), 65 deletions(-) diff --git a/tests/test_channel.py b/tests/test_channel.py index 6a85036..98b44c1 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -51,7 +51,7 @@ def test_events_channel_client_side(): assert event.identity is not None reply_event = server.new_event('someanswer', (21,), - xheader=dict(response_to=event.header['message_id'])) + xheader={b'response_to': event.header[b'message_id']}) reply_event.identity = event.identity server.emit_event(reply_event) event = client_channel.recv() @@ -78,7 +78,7 @@ def test_events_channel_client_side_server_send_many(): for x in range(10): reply_event = server.new_event('someanswer', (x,), - xheader=dict(response_to=event.header['message_id'])) + xheader={b'response_to': event.header[b'message_id']}) reply_event.identity = event.identity server.emit_event(reply_event) for x in range(10): diff --git a/tests/test_events.py b/tests/test_events.py index 91a116b..9fb4569 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -23,11 +23,9 @@ # SOFTWARE. -from __future__ import print_function -from __future__ import absolute_import -from builtins import str -from builtins import range -from builtins import object +from __future__ import print_function, absolute_import +from builtins import str, bytes +from builtins import range, object from zerorpc import zmq import zerorpc @@ -52,25 +50,25 @@ def test_event(): event = zerorpc.Event('mylittleevent', (None,), context=context) print(event) assert event.name == 'mylittleevent' - assert event.header['message_id'] == 0 + assert event.header[b'message_id'] == 0 assert event.args == (None,) event = zerorpc.Event('mylittleevent2', ('42',), context=context) print(event) assert event.name == 'mylittleevent2' - assert event.header['message_id'] == 1 + assert event.header[b'message_id'] == 1 assert event.args == ('42',) event = zerorpc.Event('mylittleevent3', ('a', 42), context=context) print(event) assert event.name == 'mylittleevent3' - assert event.header['message_id'] == 2 + assert event.header[b'message_id'] == 2 assert event.args == ('a', 42) event = zerorpc.Event('mylittleevent4', ('b', 21), context=context) print(event) assert event.name == 'mylittleevent4' - assert event.header['message_id'] == 3 + assert event.header[b'message_id'] == 3 assert event.args == ('b', 21) packed = event.pack() @@ -78,23 +76,23 @@ def test_event(): print(unpacked) assert unpacked.name == 'mylittleevent4' - assert unpacked.header['message_id'] == 3 + assert unpacked.header[b'message_id'] == 3 assert list(unpacked.args) == ['b', 21] event = zerorpc.Event('mylittleevent5', ('c', 24, True), - header={'lol': 'rofl'}, context=None) + header={b'lol': 'rofl'}, context=None) print(event) assert event.name == 'mylittleevent5' - assert event.header['lol'] == 'rofl' + assert event.header[b'lol'] == 'rofl' assert event.args == ('c', 24, True) event = zerorpc.Event('mod', (42,), context=context) print(event) assert event.name == 'mod' - assert event.header['message_id'] == 4 + assert event.header[b'message_id'] == 4 assert event.args == (42,) - event.header.update({'stream': True}) - assert event.header['stream'] is True + event.header.update({b'stream': True}) + assert event.header[b'stream'] is True def test_events_req_rep(): @@ -179,55 +177,55 @@ def test_events_push_pull(): def test_msgpack(): context = zerorpc.Context() - event = zerorpc.Event('myevent', ('a',), context=context) + event = zerorpc.Event(u'myevent', (u'a',), context=context) print(event) - assert type(event.name) == str + assert isinstance(event.name, str) for key in event.header.keys(): - assert type(key) == str - assert type(event.header['message_id']) == str - assert type(event.args[0]) == str + assert isinstance(key, bytes) + assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(event.args[0], str) packed = event.pack() event = event.unpack(packed) print(event) - assert type(event.name) == str + assert isinstance(event.name, str) for key in event.header.keys(): - assert type(key) == str - assert type(event.header['message_id']) == str - assert type(event.args[0]) == str + assert isinstance(key, bytes) + assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(event.args[0], str) - event = zerorpc.Event('myevent', (u'a',), context=context) + event = zerorpc.Event(b'myevent', (u'a',), context=context) print(event) - assert type(event.name) == str + assert isinstance(event.name, bytes) for key in event.header.keys(): - assert type(key) == str - assert type(event.header['message_id']) == str - assert type(event.args[0]) == str + assert isinstance(key, bytes) + assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(event.args[0], str) packed = event.pack() event = event.unpack(packed) print(event) - assert type(event.name) == str + assert isinstance(event.name, bytes) for key in event.header.keys(): - assert type(key) == str - assert type(event.header['message_id']) == str - assert type(event.args[0]) == str + assert isinstance(key, bytes) + assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(event.args[0], str) - event = zerorpc.Event('myevent', (u'a', 'b'), context=context) + event = zerorpc.Event(u'myevent', (u'a', u'b'), context=context) print(event) - assert type(event.name) == str + assert isinstance(event.name, str) for key in event.header.keys(): - assert type(key) == str - assert type(event.header['message_id']) == str - assert type(event.args[0]) == str - assert type(event.args[1]) == str + assert isinstance(key, bytes) + assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(event.args[0], str) + assert isinstance(event.args[1], str) packed = event.pack() event = event.unpack(packed) print(event) - assert type(event.name) == str + assert isinstance(event.name, str) for key in event.header.keys(): - assert type(key) == str - assert type(event.header['message_id']) == str - assert type(event.args[0]) == str - assert type(event.args[1]) == str + assert isinstance(key, bytes) + assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(event.args[0], str) + assert isinstance(event.args[1], str) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 5592713..754f8cb 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -26,6 +26,7 @@ from __future__ import print_function from __future__ import absolute_import from builtins import str +from future.utils import tobytes from nose.tools import assert_raises import gevent @@ -138,7 +139,7 @@ def get_task_context(self): if self.trace_id is None: # just an ugly code to generate a beautiful little hash. self._locals.trace_id = '<{0}>'.format(hashlib.md5( - str(random.random())[3:] + tobytes(str(random.random())[3:]) ).hexdigest()[0:6].upper()) print(self._identity, 'get_task_context! [make a new one]', self.trace_id) self._log.append(('new', self.trace_id)) diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index 927019d..5d48b4d 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -28,6 +28,7 @@ from builtins import range import gevent +import collections import zerorpc from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR @@ -57,7 +58,7 @@ def xrange(self, max): assert list(r) == list(range(10)) r = client.xrange(10) - assert getattr(r, 'next', None) is not None + assert isinstance(r, collections.Iterator) l = [] print('wait 4s for fun') gevent.sleep(TIME_FACTOR * 4) diff --git a/tests/test_server.py b/tests/test_server.py index 200b462..b9997b4 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -209,7 +209,7 @@ class MySrv(zerorpc.Server): assert msg == 'donotexist' rpccall = client.channel() - rpccall.emit('donotexist', tuple(), xheader=dict(v=1)) + rpccall.emit('donotexist', tuple(), xheader={b'v': 1}) event = rpccall.recv() print(event) assert event.name == 'ERR' diff --git a/tests/test_zmq.py b/tests/test_zmq.py index e3f24d2..b584cf6 100644 --- a/tests/test_zmq.py +++ b/tests/test_zmq.py @@ -42,7 +42,7 @@ def server(): r = s.recv() print('srv', r) print('srv sending...') - s.send('world') + s.send(b'world') s.close() c.term() @@ -53,7 +53,7 @@ def client(): s.connect(endpoint) print('cli sending...') - s.send('hello') + s.send(b'hello') print('cli recving...') r = s.recv() print('cli', r) diff --git a/zerorpc/channel.py b/zerorpc/channel.py index e1e6853..8e3a395 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -79,7 +79,7 @@ def _channel_dispatcher(self): except Exception: logger.exception('zerorpc.ChannelMultiplexer ignoring error on recv') continue - channel_id = event.header.get('response_to', None) + channel_id = event.header.get(b'response_to', None) queue = None if channel_id is not None: @@ -119,7 +119,7 @@ def __init__(self, multiplexer, from_event=None): self._zmqid = None self._queue = gevent.queue.Queue(maxsize=1) if from_event is not None: - self._channel_id = from_event.header['message_id'] + self._channel_id = from_event.header[b'message_id'] self._zmqid = from_event.identity self._multiplexer._active_channels[self._channel_id] = self logger.debug('<-- new channel %s', self._channel_id) @@ -142,11 +142,11 @@ def close(self): def new_event(self, name, args, xheader=None): event = self._multiplexer.new_event(name, args, xheader) if self._channel_id is None: - self._channel_id = event.header['message_id'] + self._channel_id = event.header[b'message_id'] self._multiplexer._active_channels[self._channel_id] = self logger.debug('--> new channel %s', self._channel_id) else: - event.header['response_to'] = self._channel_id + event.header[b'response_to'] = self._channel_id event.identity = self._zmqid return event diff --git a/zerorpc/context.py b/zerorpc/context.py index 9600d62..debce26 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -24,7 +24,7 @@ from __future__ import absolute_import -from builtins import str +from future.utils import tobytes import uuid import random @@ -103,7 +103,7 @@ def get_instance(): return Context._instance def _reset_msgid(self): - self._msg_id_base = str(uuid.uuid4())[8:] + self._msg_id_base = tobytes(uuid.uuid4().hex)[8:] self._msg_id_counter = random.randrange(0, 2 ** 32) self._msg_id_counter_stop = random.randrange(self._msg_id_counter, 2 ** 32) @@ -112,7 +112,7 @@ def new_msgid(self): self._reset_msgid() else: self._msg_id_counter = (self._msg_id_counter + 1) - return '{0:08x}{1}'.format(self._msg_id_counter, self._msg_id_base) + return tobytes('{0:08x}'.format(self._msg_id_counter)) + self._msg_id_base def register_middleware(self, middleware_instance): registered_count = 0 diff --git a/zerorpc/core.py b/zerorpc/core.py index 72e394e..41e435c 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -138,7 +138,7 @@ def _print_traceback(self, protocol_v1, exc_infos): return (name, human_msg, human_traceback) def _async_task(self, initial_event): - protocol_v1 = initial_event.header.get('v', 1) < 2 + protocol_v1 = initial_event.header.get(b'v', 1) < 2 channel = self._multiplexer.channel(initial_event) hbchan = HeartBeatOnChannel(channel, freq=self._heartbeat_freq, passive=protocol_v1) @@ -201,7 +201,7 @@ def close(self): def _handle_remote_error(self, event): exception = self._context.hook_client_handle_remote_error(event) if not exception: - if event.header.get('v', 1) >= 2: + if event.header.get(b'v', 1) >= 2: (name, msg, traceback) = event.args exception = RemoteError(name, msg, traceback) else: @@ -370,7 +370,7 @@ class Subscriber(Puller): def __init__(self, methods=None, context=None): super(Subscriber, self).__init__(methods=methods, context=context, zmq_socket=zmq.SUB) - self._events.setsockopt(zmq.SUBSCRIBE, '') + self._events.setsockopt(zmq.SUBSCRIBE, b'') def fork_task_context(functor, context=None): diff --git a/zerorpc/events.py b/zerorpc/events.py index 07f3266..7bfa6bf 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -49,6 +49,10 @@ def get_pyzmq_frame_buffer(frame): def get_pyzmq_frame_buffer(frame): return frame.buffer +# gevent <= 1.1.0.rc5 is missing the Python3 __next__ method. +if sys.version_info >= (3, 0) and gevent.version_info <= (1, 1, 0, 'rc', '5'): + setattr(gevent.queue.Channel, '__next__', gevent.queue.Channel.next) + logger = logging.getLogger(__name__) @@ -166,7 +170,7 @@ def __init__(self, name, args, context, header=None): self._name = name self._args = args if header is None: - self._header = {'message_id': context.new_msgid(), 'v': 3} + self._header = {b'message_id': context.new_msgid(), b'v': 3} else: self._header = header self._identity = None @@ -334,9 +338,9 @@ def emit_event(self, event, timeout=None): logger.debug('--> %s', event) if event.identity: parts = list(event.identity or list()) - parts.extend(['', event.pack()]) + parts.extend([b'', event.pack()]) elif self._zmq_socket_type in (zmq.DEALER, zmq.ROUTER): - parts = ('', event.pack()) + parts = (b'', event.pack()) else: parts = (event.pack(),) self._send(parts, timeout) diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index 5ac9206..776358d 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -91,7 +91,7 @@ def _recver(self): while True: event = self._channel.recv() if self._compat_v2 is None: - self._compat_v2 = event.header.get('v', 0) < 3 + self._compat_v2 = event.header.get(b'v', 0) < 3 if event.name == '_zpc_hb': self._remote_last_hb = time.time() self._start_heartbeat() From 294e7c7e980ba8340c08c0fe56f9802291a5a652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 19:44:43 -0700 Subject: [PATCH 081/113] Add python 3.5 to travis. Also reduce the test matrix on PYZMQ. --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc548af..79ce4f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,12 @@ python: - 2.6 - 2.7 - 3.4 + - 3.5 env: matrix: - - PYZMQ='pyzmq>=14' - - PYZMQ='pyzmq>=14.3' - - PYZMQ='pyzmq>=14.2,<14.3' - - PYZMQ='pyzmq>=14.1,<14.2' - - PYZMQ='pyzmq>=14.0,<14.1' - - PYZMQ='pyzmq<14' + - PYZMQ='pyzmq>=15' + - PYZMQ='pyzmq>=14,<15' + - PYZMQ='pyzmq>=13,<14' matrix: fast_finish: true script: From 1c55df981a81df0e8f8f05c034ba8b4f701302dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 20 Mar 2016 21:27:18 -0700 Subject: [PATCH 082/113] Fix travis config to install dependencies properly. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79ce4f4..9073da8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,5 +22,5 @@ addons: - libevent-dev install: - pip install flake8 - - "pip install nose gevent msgpack-python $PYZMQ" - - pip install . --no-deps + - "pip install nose $PYZMQ" + - pip install . From d67b81daf8d82b147c9ae9ed8633b868508c17ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 27 May 2016 19:32:01 -0700 Subject: [PATCH 083/113] Tox config updated. --- tox.ini | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index b56bac5..22e50ab 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,15 @@ [tox] -envlist = py26,py27,py31,py32 +envlist = py26,py27,py34,py35 [testenv] deps = flake8 nose commands = - flake8 --ignore=E501,E128 zerorpc bin - nosetests + flake8 zerorpc bin + nosetests -v +setenv = + ZPC_TEST_TIME_FACTOR=0.2 [flake8] ignore = E501,E128 From c475621c9ec3e7230a99e1a2fae2fb645c1fca6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 27 May 2016 19:56:51 -0700 Subject: [PATCH 084/113] New default time factor is 0.2 --- .travis.yml | 2 +- tests/testutils.py | 4 ++-- tox.ini | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9073da8..3399b99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ matrix: fast_finish: true script: - flake8 --ignore=E501,E128 zerorpc bin - - ZPC_TEST_TIME_FACTOR=0.2 nosetests -v + - nosetests -v sudo: false addons: apt: diff --git a/tests/testutils.py b/tests/testutils.py index eb731ce..2a0110c 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -58,6 +58,6 @@ def wrap(): try: TIME_FACTOR = float(os.environ.get('ZPC_TEST_TIME_FACTOR')) - print('ZPC_TEST_TIME_FACTOR:', TIME_FACTOR) except TypeError: - TIME_FACTOR = 1.0 + TIME_FACTOR = 0.2 +print('ZPC_TEST_TIME_FACTOR:', TIME_FACTOR) diff --git a/tox.ini b/tox.ini index 22e50ab..3490b2c 100644 --- a/tox.ini +++ b/tox.ini @@ -8,8 +8,7 @@ deps = commands = flake8 zerorpc bin nosetests -v -setenv = - ZPC_TEST_TIME_FACTOR=0.2 +passenv = ZPC_TEST_TIME_FACTOR [flake8] ignore = E501,E128 From a5530875abd77f81d2c3694ec4d309c7cb9f6ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 27 May 2016 19:58:01 -0700 Subject: [PATCH 085/113] Ignore GreenletExit exceptions in Events.close(). During a gc pass, children greenlets might have exited already. --- zerorpc/events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zerorpc/events.py b/zerorpc/events.py index 7bfa6bf..3437748 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -275,11 +275,11 @@ def __del__(self): def close(self): try: self._send.close() - except AttributeError: + except (AttributeError, TypeError, gevent.GreenletExit): pass try: self._recv.close() - except AttributeError: + except (AttributeError, TypeError, gevent.GreenletExit): pass self._socket.close() From 6c9c7364d29f062c6791c140ccce363e6f1a4248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 3 Jun 2016 21:44:15 -0700 Subject: [PATCH 086/113] Finally correctly handling msgpack bin vs string. A long time ago msgpack had a single RAW type for bytes string. It was the application responsibility to use any compatible string encoding across languages/runtimes. At some point msgpack was updated with a distinction between BIN type and STRING type. The STRING type is the RAW type renamed, and should only contain and UTF-8 encoded string. The BIN type is to contain any byte strings encoded at the application's responsibility. Because STRING is the RAW type renamed, when reading a msgpack message written an older version of msgpack, it is impossible to know if the string was supposed to be encoded in UTF-8 or not. So msgpack-python for backward compatibility, reads both RAW and BIN by as bytes strings by default. On the write side bytes strings and unicode strings encoded to UTF-8 are sent as STRING (and it will appear as UTF-8 string on nodejs for example). And btw Python2 strings are somewhat loosely defined ASCII strings (which is a subset of UTF-8) so everything was all fine, in/out of zerorpc-python was UTF-8 strings. But one day, somebody wanted zerorpc-python to handle the distinction between unicode strings and bytes strings. This was the day commit 6ca6fdf598905ca67f2678973034dae435a28594 was born. This broke the compatibility with zerorpc-node because suddenly, zerorpc-python on python2 (remember, strings on python2 are bytes strings) was sending BIN (bytes strings) instead of STRING (utf-8 strings) that zerorpc-node expects. On the other hand, zerorpc-python on python3 was sending STRING as it should, because in python3, strings are unicode by default and msgpack-python will encode them as UTF-8 strings with the type STRING. In an attempt of making zerorpc-python working across python2 & python3, I did the wrong thing and used Python2 as reference instead of using Python3, which further broke compatibility with zerorpc-node. This commit restores compatibility with zerorpc-node and across python3/python3. Fixes #142. Updates #108. Note also this new repo 0rpc/zerorpc-crosstests to run basic zerorpc tests across languages. I hope it will prevent this kind of bugs to be published ever again. --- tests/test_channel.py | 4 +-- tests/test_events.py | 74 ++++++++++++------------------------------- tests/test_server.py | 2 +- tests/test_zmq.py | 2 +- zerorpc/channel.py | 12 +++---- zerorpc/core.py | 23 ++++++++++++-- zerorpc/events.py | 10 ++++-- zerorpc/heartbeat.py | 12 +++---- zerorpc/patterns.py | 22 ++++++------- 9 files changed, 76 insertions(+), 85 deletions(-) diff --git a/tests/test_channel.py b/tests/test_channel.py index 98b44c1..1d59b1e 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -51,7 +51,7 @@ def test_events_channel_client_side(): assert event.identity is not None reply_event = server.new_event('someanswer', (21,), - xheader={b'response_to': event.header[b'message_id']}) + xheader={'response_to': event.header['message_id']}) reply_event.identity = event.identity server.emit_event(reply_event) event = client_channel.recv() @@ -78,7 +78,7 @@ def test_events_channel_client_side_server_send_many(): for x in range(10): reply_event = server.new_event('someanswer', (x,), - xheader={b'response_to': event.header[b'message_id']}) + xheader={'response_to': event.header['message_id']}) reply_event.identity = event.identity server.emit_event(reply_event) for x in range(10): diff --git a/tests/test_events.py b/tests/test_events.py index 9fb4569..7acf98e 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -50,49 +50,49 @@ def test_event(): event = zerorpc.Event('mylittleevent', (None,), context=context) print(event) assert event.name == 'mylittleevent' - assert event.header[b'message_id'] == 0 + assert event.header['message_id'] == 0 assert event.args == (None,) event = zerorpc.Event('mylittleevent2', ('42',), context=context) print(event) assert event.name == 'mylittleevent2' - assert event.header[b'message_id'] == 1 + assert event.header['message_id'] == 1 assert event.args == ('42',) event = zerorpc.Event('mylittleevent3', ('a', 42), context=context) print(event) assert event.name == 'mylittleevent3' - assert event.header[b'message_id'] == 2 + assert event.header['message_id'] == 2 assert event.args == ('a', 42) - event = zerorpc.Event('mylittleevent4', ('b', 21), context=context) + event = zerorpc.Event('mylittleevent4', ('', 21), context=context) print(event) assert event.name == 'mylittleevent4' - assert event.header[b'message_id'] == 3 - assert event.args == ('b', 21) + assert event.header['message_id'] == 3 + assert event.args == ('', 21) packed = event.pack() unpacked = zerorpc.Event.unpack(packed) print(unpacked) assert unpacked.name == 'mylittleevent4' - assert unpacked.header[b'message_id'] == 3 - assert list(unpacked.args) == ['b', 21] + assert unpacked.header['message_id'] == 3 + assert list(unpacked.args) == ['', 21] event = zerorpc.Event('mylittleevent5', ('c', 24, True), - header={b'lol': 'rofl'}, context=None) + header={'lol': 'rofl'}, context=None) print(event) assert event.name == 'mylittleevent5' - assert event.header[b'lol'] == 'rofl' + assert event.header['lol'] == 'rofl' assert event.args == ('c', 24, True) event = zerorpc.Event('mod', (42,), context=context) print(event) assert event.name == 'mod' - assert event.header[b'message_id'] == 4 + assert event.header['message_id'] == 4 assert event.args == (42,) - event.header.update({b'stream': True}) - assert event.header[b'stream'] is True + event.header.update({'stream': True}) + assert event.header['stream'] is True def test_events_req_rep(): @@ -179,10 +179,13 @@ def test_msgpack(): context = zerorpc.Context() event = zerorpc.Event(u'myevent', (u'a',), context=context) print(event) + # note here that str is an unicode string in all Python version (thanks to + # the builtin str import). assert isinstance(event.name, str) for key in event.header.keys(): - assert isinstance(key, bytes) - assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(key, str) + assert isinstance(event.header[u'message_id'], bytes) + assert isinstance(event.header[u'v'], int) assert isinstance(event.args[0], str) packed = event.pack() @@ -190,42 +193,7 @@ def test_msgpack(): print(event) assert isinstance(event.name, str) for key in event.header.keys(): - assert isinstance(key, bytes) - assert isinstance(event.header[b'message_id'], bytes) + assert isinstance(key, str) + assert isinstance(event.header[u'message_id'], bytes) + assert isinstance(event.header[u'v'], int) assert isinstance(event.args[0], str) - - event = zerorpc.Event(b'myevent', (u'a',), context=context) - print(event) - assert isinstance(event.name, bytes) - for key in event.header.keys(): - assert isinstance(key, bytes) - assert isinstance(event.header[b'message_id'], bytes) - assert isinstance(event.args[0], str) - - packed = event.pack() - event = event.unpack(packed) - print(event) - assert isinstance(event.name, bytes) - for key in event.header.keys(): - assert isinstance(key, bytes) - assert isinstance(event.header[b'message_id'], bytes) - assert isinstance(event.args[0], str) - - event = zerorpc.Event(u'myevent', (u'a', u'b'), context=context) - print(event) - assert isinstance(event.name, str) - for key in event.header.keys(): - assert isinstance(key, bytes) - assert isinstance(event.header[b'message_id'], bytes) - assert isinstance(event.args[0], str) - assert isinstance(event.args[1], str) - - packed = event.pack() - event = event.unpack(packed) - print(event) - assert isinstance(event.name, str) - for key in event.header.keys(): - assert isinstance(key, bytes) - assert isinstance(event.header[b'message_id'], bytes) - assert isinstance(event.args[0], str) - assert isinstance(event.args[1], str) diff --git a/tests/test_server.py b/tests/test_server.py index b9997b4..2a266ce 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -209,7 +209,7 @@ class MySrv(zerorpc.Server): assert msg == 'donotexist' rpccall = client.channel() - rpccall.emit('donotexist', tuple(), xheader={b'v': 1}) + rpccall.emit('donotexist', tuple(), xheader={'v': 1}) event = rpccall.recv() print(event) assert event.name == 'ERR' diff --git a/tests/test_zmq.py b/tests/test_zmq.py index b584cf6..1e7b4dd 100644 --- a/tests/test_zmq.py +++ b/tests/test_zmq.py @@ -38,7 +38,7 @@ def server(): s = c.socket(zmq.REP) s.bind(endpoint) while True: - print('srv recving...') + print(b'srv recving...') r = s.recv() print('srv', r) print('srv sending...') diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 8e3a395..ad21c27 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -79,7 +79,7 @@ def _channel_dispatcher(self): except Exception: logger.exception('zerorpc.ChannelMultiplexer ignoring error on recv') continue - channel_id = event.header.get(b'response_to', None) + channel_id = event.header.get(u'response_to', None) queue = None if channel_id is not None: @@ -119,7 +119,7 @@ def __init__(self, multiplexer, from_event=None): self._zmqid = None self._queue = gevent.queue.Queue(maxsize=1) if from_event is not None: - self._channel_id = from_event.header[b'message_id'] + self._channel_id = from_event.header[u'message_id'] self._zmqid = from_event.identity self._multiplexer._active_channels[self._channel_id] = self logger.debug('<-- new channel %s', self._channel_id) @@ -142,11 +142,11 @@ def close(self): def new_event(self, name, args, xheader=None): event = self._multiplexer.new_event(name, args, xheader) if self._channel_id is None: - self._channel_id = event.header[b'message_id'] + self._channel_id = event.header[u'message_id'] self._multiplexer._active_channels[self._channel_id] = self logger.debug('--> new channel %s', self._channel_id) else: - event.header[b'response_to'] = self._channel_id + event.header[u'response_to'] = self._channel_id event.identity = self._zmqid return event @@ -205,7 +205,7 @@ def close(self): def _recver(self): while True: event = self._channel.recv() - if event.name == '_zpc_more': + if event.name == u'_zpc_more': try: self._remote_queue_open_slots += int(event.args[0]) except Exception: @@ -239,7 +239,7 @@ def emit_event(self, event, timeout=None): def _request_data(self): open_slots = self._input_queue_size - self._input_queue_reserved self._input_queue_reserved += open_slots - self._channel.emit('_zpc_more', (open_slots,)) + self._channel.emit(u'_zpc_more', (open_slots,)) def recv(self, timeout=None): # self._channel can be set to None by an 'on_close_if' callback if it diff --git a/zerorpc/core.py b/zerorpc/core.py index 41e435c..9fa63e3 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -138,7 +138,7 @@ def _print_traceback(self, protocol_v1, exc_infos): return (name, human_msg, human_traceback) def _async_task(self, initial_event): - protocol_v1 = initial_event.header.get(b'v', 1) < 2 + protocol_v1 = initial_event.header.get(u'v', 1) < 2 channel = self._multiplexer.channel(initial_event) hbchan = HeartBeatOnChannel(channel, freq=self._heartbeat_freq, passive=protocol_v1) @@ -157,7 +157,7 @@ def _async_task(self, initial_event): except Exception: exc_infos = list(sys.exc_info()) human_exc_infos = self._print_traceback(protocol_v1, exc_infos) - reply_event = bufchan.new_event('ERR', human_exc_infos, + reply_event = bufchan.new_event(u'ERR', human_exc_infos, self._context.hook_get_task_context()) self._context.hook_server_inspect_exception(event, reply_event, exc_infos) bufchan.emit_event(reply_event) @@ -201,7 +201,7 @@ def close(self): def _handle_remote_error(self, event): exception = self._context.hook_client_handle_remote_error(event) if not exception: - if event.header.get(b'v', 1) >= 2: + if event.header.get(u'v', 1) >= 2: (name, msg, traceback) = event.args exception = RemoteError(name, msg, traceback) else: @@ -238,6 +238,23 @@ def raise_error(ex): reply_event, self._handle_remote_error) def __call__(self, method, *args, **kargs): + # here `method` is either a string of bytes or an unicode string in + # Python2 and Python3. Python2: str aka a byte string containing ASCII + # (unless the user explicitly provide an unicode string). Python3: str + # aka an unicode string (unless the user explicitly provide a byte + # string). + # zerorpc protocol requires an utf-8 encoded string at the msgpack + # level. msgpack will encode any unicode string object to UTF-8 and tag + # it `string`, while a bytes string will be tagged `bin`. + # + # So when we get a bytes string, we assume it to be an UTF-8 string + # (ASCII is contained in UTF-8) that we decode to an unicode string. + # Right after, msgpack-python will re-encode it as UTF-8. Yes this is + # terribly inefficient with Python2 because most of the time `method` + # will already be an UTF-8 encoded bytes string. + if isinstance(method, bytes): + method = method.decode('utf-8') + timeout = kargs.get('timeout', self._timeout) channel = self._multiplexer.channel() hbchan = HeartBeatOnChannel(channel, freq=self._heartbeat_freq, diff --git a/zerorpc/events.py b/zerorpc/events.py index 3437748..cd75bfe 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -166,11 +166,15 @@ class Event(object): __slots__ = ['_name', '_args', '_header', '_identity'] + # protocol details: + # - `name` and `header` keys must be unicode strings. + # - `message_id` and 'response_to' values are opaque bytes string. + # - `v' value is an integer. def __init__(self, name, args, context, header=None): self._name = name self._args = args if header is None: - self._header = {b'message_id': context.new_msgid(), b'v': 3} + self._header = {u'message_id': context.new_msgid(), u'v': 3} else: self._header = header self._identity = None @@ -200,7 +204,9 @@ def identity(self, v): self._identity = v def pack(self): - return msgpack.Packer(use_bin_type=True).pack((self._header, self._name, self._args)) + payload = (self._header, self._name, self._args) + r = msgpack.Packer(use_bin_type=True).pack(payload) + return r @staticmethod def unpack(blob): diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index 776358d..f0f317e 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -81,7 +81,7 @@ def _heartbeat(self): gevent.kill(self._parent_coroutine, self._lost_remote_exception()) break - self._channel.emit('_zpc_hb', (0,)) # 0 -> compat with protocol v2 + self._channel.emit(u'_zpc_hb', (0,)) # 0 -> compat with protocol v2 def _start_heartbeat(self): if self._heartbeat_task is None and self._heartbeat_freq is not None and not self._closed: @@ -91,12 +91,12 @@ def _recver(self): while True: event = self._channel.recv() if self._compat_v2 is None: - self._compat_v2 = event.header.get(b'v', 0) < 3 - if event.name == '_zpc_hb': + self._compat_v2 = event.header.get(u'v', 0) < 3 + if event.name == u'_zpc_hb': self._remote_last_hb = time.time() self._start_heartbeat() if self._compat_v2: - event.name = '_zpc_more' + event.name = u'_zpc_more' self._input_queue.put(event) else: self._input_queue.put(event) @@ -106,8 +106,8 @@ def _lost_remote_exception(self): self._heartbeat_freq * 2)) def new_event(self, name, args, header=None): - if self._compat_v2 and name == '_zpc_more': - name = '_zpc_hb' + if self._compat_v2 and name == u'_zpc_more': + name = u'_zpc_hb' return self._channel.new_event(name, args, header) def emit_event(self, event, timeout=None): diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index 8d6ee0a..3623e17 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -28,18 +28,18 @@ class ReqRep(object): def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) result = functor(*req_event.args) - rep_event = channel.new_event('OK', (result,), + rep_event = channel.new_event(u'OK', (result,), context.hook_get_task_context()) context.hook_server_after_exec(req_event, rep_event) channel.emit_event(rep_event) def accept_answer(self, event): - return event.name in ('OK', 'ERR') + return event.name in (u'OK', u'ERR') def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): try: - if rep_event.name == 'ERR': + if rep_event.name == u'ERR': exception = handle_remote_error(rep_event) context.hook_client_after_request(req_event, rep_event, exception) raise exception @@ -55,33 +55,33 @@ def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) xheader = context.hook_get_task_context() for result in iter(functor(*req_event.args)): - channel.emit('STREAM', result, xheader) - done_event = channel.new_event('STREAM_DONE', None, xheader) + channel.emit(u'STREAM', result, xheader) + done_event = channel.new_event(u'STREAM_DONE', None, xheader) # NOTE: "We" made the choice to call the hook once the stream is done, - # the other choice was to call it at each iteration. I don't think that - # one choice is better than the other, so I'm fine with changing this + # the other choice was to call it at each iteration. I donu't think that + # one choice is better than the other, so Iu'm fine with changing this # or adding the server_after_iteration and client_after_iteration hooks. context.hook_server_after_exec(req_event, done_event) channel.emit_event(done_event) def accept_answer(self, event): - return event.name in ('STREAM', 'STREAM_DONE') + return event.name in (u'STREAM', u'STREAM_DONE') def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): def is_stream_done(rep_event): - return rep_event.name == 'STREAM_DONE' + return rep_event.name == u'STREAM_DONE' channel.on_close_if = is_stream_done def iterator(req_event, rep_event): try: - while rep_event.name == 'STREAM': + while rep_event.name == u'STREAM': # Like in process_call, we made the choice to call the # after_exec hook only when the stream is done. yield rep_event.args rep_event = channel.recv() - if rep_event.name == 'ERR': + if rep_event.name == u'ERR': exception = handle_remote_error(rep_event) context.hook_client_after_request(req_event, rep_event, exception) raise exception From eb2b4006a78dea1e56ca2b9e13850e5572b19297 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Wed, 31 Aug 2016 17:41:18 +0200 Subject: [PATCH 087/113] fixes encoding problem --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b68b42e..ff335a4 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ # SOFTWARE. # execfile() doesn't exist in Python 3, this way we are compatible with both. -exec(compile(open('zerorpc/version.py').read(), 'zerorpc/version.py', 'exec')) +exec(compile(open('zerorpc/version.py', encoding='utf8').read(), 'zerorpc/version.py', 'exec')) import sys From 45c0015b39e50882db4b123f700a373883052cd8 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Wed, 31 Aug 2016 19:06:36 +0200 Subject: [PATCH 088/113] make open() with encoding work under Python2 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index ff335a4..352a96c 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from io import open + # execfile() doesn't exist in Python 3, this way we are compatible with both. exec(compile(open('zerorpc/version.py', encoding='utf8').read(), 'zerorpc/version.py', 'exec')) From 47c031dc9a39c59f3c11b99e486403c08a6395ad Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Thu, 1 Sep 2016 10:46:48 +0200 Subject: [PATCH 089/113] trying yet another execfile alternative --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 352a96c..887892b 100644 --- a/setup.py +++ b/setup.py @@ -22,10 +22,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from io import open +from past.builtins import execfile -# execfile() doesn't exist in Python 3, this way we are compatible with both. -exec(compile(open('zerorpc/version.py', encoding='utf8').read(), 'zerorpc/version.py', 'exec')) +execfile('zerorpc/version.py') import sys From b34abce59eab1f30cf7759d43622a52eb02e9208 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Thu, 1 Sep 2016 10:58:27 +0200 Subject: [PATCH 090/113] adding a special case for Python3 --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 887892b..6644e96 100644 --- a/setup.py +++ b/setup.py @@ -22,12 +22,12 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from past.builtins import execfile - -execfile('zerorpc/version.py') - import sys +if sys.version_info < (3, 0): + execfile('zerorpc/version.py') +else: + exec(compile(open('zerorpc/version.py', encoding='utf8').read(), 'zerorpc/version.py', 'exec')) try: from setuptools import setup From 12cb7633dc00c9f91b66ca379a5240ce8b677867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 4 Sep 2016 07:46:40 -0700 Subject: [PATCH 091/113] Explicit imports to please flake8 Let's be honest, its also cleaner. --- zerorpc/decorators.py | 2 +- zerorpc/gevent_zmq.py | 3 +++ zerorpc/heartbeat.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/zerorpc/decorators.py b/zerorpc/decorators.py index 60cf1f8..8ef39dc 100644 --- a/zerorpc/decorators.py +++ b/zerorpc/decorators.py @@ -24,7 +24,7 @@ import inspect -from .patterns import * # noqa +from .patterns import ReqRep, ReqStream class DecoratorBase(object): diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index badd8ea..9430695 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -27,6 +27,9 @@ # We want to act like zmq from zmq import * # noqa +# Explicit import to please flake8 +from zmq import ZMQError + # A way to access original zmq import zmq as _zmq diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index f0f317e..23b974d 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -30,7 +30,7 @@ import gevent.local import gevent.lock -from .exceptions import * # noqa +from .exceptions import LostRemote, TimeoutExpired from .channel_base import ChannelBase From beb8606c263c1e855601f59cd808f769c748bf88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 4 Sep 2016 08:05:33 -0700 Subject: [PATCH 092/113] Do not run flake8 on travis for python2.6. flake8 doesn't support python2.6 anymore. --- .travis.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3399b99..4a59b0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +sudo: false language: python python: - 2.6 @@ -11,16 +12,19 @@ env: - PYZMQ='pyzmq>=13,<14' matrix: fast_finish: true -script: - - flake8 --ignore=E501,E128 zerorpc bin - - nosetests -v -sudo: false addons: apt: packages: - python-dev - libevent-dev install: - - pip install flake8 + - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then + pip install flake8; + fi - "pip install nose $PYZMQ" - pip install . +script: + - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then + flake8 --ignore=E501,E128 zerorpc bin; + fi + - nosetests -v From e0c6107ad39c3ccafe7fd5d929214eb7895762db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Sun, 4 Sep 2016 08:52:31 -0700 Subject: [PATCH 093/113] Bump version to 0.6.0. --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index 00742c9..4a455ec 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.5.2' +__version__ = '0.6.0' __author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' __copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From 05319212da3bcce812368773e8c85b05883b2c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Tue, 20 Sep 2016 11:17:57 -0700 Subject: [PATCH 094/113] Update README.rst --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index d2b2965..93b43ef 100644 --- a/README.rst +++ b/README.rst @@ -16,6 +16,14 @@ with a convenient script, "zerorpc", allowing to: * expose Python modules without modifying a single line of code, * call those modules remotely through the command line. +Installation +------------ + +On most systems, its a matter of:: + + $ pip install zerorpc + +Depending of the support from Gevent and PyZMQ on your system, you might need to install `libev` (for gevent) and `libzmq` (for pyzmq) with the development files. Create a server with a one-liner -------------------------------- From 08b87639ac5148f85a885d8e88b04d35214613db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Mon, 30 Jan 2017 17:11:03 -0800 Subject: [PATCH 095/113] gevent 1.1.* for python 2.6 --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6644e96..b88b308 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,9 @@ if sys.version_info < (2, 7): requirements.append('argparse') -if sys.version_info < (3, 0): +if sys.version_info < (2, 7): + requirements.append('gevent==1.1.*') +elif sys.version_info < (3, 0): requirements.append('gevent>=1.0') else: requirements.append('gevent>=1.1rc5') From eb8da609bb504a684d891f99e00298e482144e51 Mon Sep 17 00:00:00 2001 From: Samuel Killin Date: Thu, 16 Mar 2017 07:38:14 +1100 Subject: [PATCH 096/113] Expose pool_size on CLI --- zerorpc/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zerorpc/cli.py b/zerorpc/cli.py index 2985c91..c0211d2 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -61,6 +61,8 @@ parser.add_argument('--heartbeat', default=5, metavar='seconds', type=int, help='heartbeat frequency. You should always use \ the same frequency as the server. (default: 5s)') +parser.add_argument('--pool-size', default=None, metavar='count', type=int, + help='size of worker pool. --server only.') parser.add_argument('-j', '--json', default=False, action='store_true', help='arguments are in JSON format and will be be parsed \ before being sent to the remote') @@ -116,7 +118,7 @@ def run_server(args): if callable(server_obj): server_obj = server_obj() - server = zerorpc.Server(server_obj, heartbeat=args.heartbeat) + server = zerorpc.Server(server_obj, heartbeat=args.heartbeat, pool_size=args.pool_size) if args.debug: server.debug = True setup_links(args, server) From fbba28b9f44068accc96f25c4c99240595c631bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Thu, 13 Apr 2017 20:35:28 -0700 Subject: [PATCH 097/113] Bump version to 0.6.1. --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index 4a455ec..99b5366 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.6.0' +__version__ = '0.6.1' __author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' __copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From a4f954766679ad7554325f06a2fdf20563a1768d Mon Sep 17 00:00:00 2001 From: gtt116 Date: Thu, 1 Jun 2017 21:35:19 +0800 Subject: [PATCH 098/113] Fix setup.py in python2.6 and setuptools 0.6 In python2.6, the setuptools version is '0.6' which do not support wild-card in version. The result is pip install failed. This patch fix it. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b88b308..a54f792 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ requirements.append('argparse') if sys.version_info < (2, 7): - requirements.append('gevent==1.1.*') + requirements.append('gevent>=1.1.0,<1.2.0') elif sys.version_info < (3, 0): requirements.append('gevent>=1.0') else: From dbdbbc8114d3a73c6295233f9e8ba62e31250113 Mon Sep 17 00:00:00 2001 From: Alex Long Date: Mon, 28 Aug 2017 11:52:21 -0400 Subject: [PATCH 099/113] If functor's __name__ is not available then just stringify it --- zerorpc/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/decorators.py b/zerorpc/decorators.py index 8ef39dc..43dfa64 100644 --- a/zerorpc/decorators.py +++ b/zerorpc/decorators.py @@ -33,7 +33,7 @@ class DecoratorBase(object): def __init__(self, functor): self._functor = functor self.__doc__ = functor.__doc__ - self.__name__ = functor.__name__ + self.__name__ = getattr(functor, "__name__", str(functor)) def __get__(self, instance, type_instance=None): if instance is None: From 660891e4575d35950874d05913567521bb87ee42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Fri, 2 Feb 2018 13:03:37 -0800 Subject: [PATCH 100/113] msgpack-python was renamed to msgpack As of version 0.5 msgpack-python was renamed mgspack Fixes #185 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a54f792..8206147 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ requirements = [ - 'msgpack-python>=0.4.0', + 'msgpack>=0.5.2', 'pyzmq>=13.1.0', 'future', ] From 516b9b8d5b192a1347405f5839bb5a5460f3ecdb Mon Sep 17 00:00:00 2001 From: David Antliff Date: Wed, 12 Dec 2018 16:44:35 +1300 Subject: [PATCH 101/113] MsgPack is deprecating Unpacker(encoding='utf-8') - use 'raw=True' as recommended by MsgPack docs. --- zerorpc/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/events.py b/zerorpc/events.py index 6939165..f87d0b5 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -210,7 +210,7 @@ def pack(self): @staticmethod def unpack(blob): - unpacker = msgpack.Unpacker(encoding='utf-8') + unpacker = msgpack.Unpacker(raw=False) unpacker.feed(blob) unpacked_msg = unpacker.unpack() From 5cf189d55547fc106d25a2ee1c9f06ac679044a2 Mon Sep 17 00:00:00 2001 From: Shiplu Mokaddim Date: Thu, 13 Jun 2019 14:42:49 +0200 Subject: [PATCH 102/113] Put non-output to sys.stderr --- zerorpc/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zerorpc/cli.py b/zerorpc/cli.py index c0211d2..2907c25 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -92,7 +92,7 @@ def setup_links(args, socket): if args.bind: for endpoint in args.bind: - print('binding to "{0}"'.format(endpoint)) + print('binding to "{0}"'.format(endpoint), file=sys.stderr) socket.bind(endpoint) addresses = [] if args.address: @@ -100,7 +100,7 @@ def setup_links(args, socket): if args.connect: addresses.extend(args.connect) for endpoint in addresses: - print('connecting to "{0}"'.format(endpoint)) + print('connecting to "{0}"'.format(endpoint), file=sys.stderr) socket.connect(endpoint) @@ -122,7 +122,7 @@ def run_server(args): if args.debug: server.debug = True setup_links(args, server) - print('serving "{0}"'.format(server_obj_path)) + print('serving "{0}"'.format(server_obj_path), file=sys.stderr) return server.run() From eedfb85b0e8b30627e40364bed32dff004e6077f Mon Sep 17 00:00:00 2001 From: himanshu4code <52200766+himanshu4code@users.noreply.github.com> Date: Tue, 25 Jun 2019 15:59:35 +0530 Subject: [PATCH 103/113] updated gevent version for python 3.6 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8206147..730b19f 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ elif sys.version_info < (3, 0): requirements.append('gevent>=1.0') else: - requirements.append('gevent>=1.1rc5') + requirements.append('gevent>=1.1') setup( From d28a744330744234a11e222d88fb427aeb1ad32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Wed, 26 Jun 2019 02:00:13 -0700 Subject: [PATCH 104/113] Bump version to 0.6.2. --- zerorpc/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zerorpc/version.py b/zerorpc/version.py index 99b5366..32b8c89 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.6.1' +__version__ = '0.6.2' __author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' __copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From 57451b80ef74c119a30e1a947f61b189f07dfcd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-Xavier=20Bourlet?= Date: Wed, 26 Jun 2019 02:00:19 -0700 Subject: [PATCH 105/113] Bump version to 0.6.3. Fix README.rst. Export long_description in setup.py. This is to upload to pypi with twine. --- README.rst | 4 ++-- setup.py | 4 ++++ zerorpc/version.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 93b43ef..3538015 100644 --- a/README.rst +++ b/README.rst @@ -36,7 +36,7 @@ we will expose the Python "time" module:: .. note:: The bind address uses the zeromq address format. You are not limited to TCP transport: you could as well specify ipc:///tmp/time to use - host-local sockets, for instance. "tcp://*:1234" is a short-hand to + host-local sockets, for instance. "tcp://\*:1234" is a short-hand to "tcp://0.0.0.0:1234" and means "listen on TCP port 1234, accepting connections on all IP addresses". @@ -137,7 +137,7 @@ the "--bind" option:: $ zerorpc --server --bind tcp://*:1234 --bind ipc:///tmp/time time -You can then connect to it using either "zerorpc tcp://*:1234" or +You can then connect to it using either "zerorpc tcp://\*:1234" or "zerorpc ipc:///tmp/time". Wait, there is more! You can even mix "--bind" and "--connect". That means diff --git a/setup.py b/setup.py index 730b19f..610e68b 100644 --- a/setup.py +++ b/setup.py @@ -51,11 +51,15 @@ else: requirements.append('gevent>=1.1') +with open("README.rst", "r") as fh: + long_description = fh.read() setup( name='zerorpc', version=__version__, description='zerorpc is a flexible RPC based on zeromq.', + long_description=long_description, + long_description_content_type='text/x-rst', author=__author__, url='https://github.com/0rpc/zerorpc-python', packages=['zerorpc'], diff --git a/zerorpc/version.py b/zerorpc/version.py index 32b8c89..4324a5a 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.6.2' +__version__ = '0.6.3' __author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' __copyright__ = 'Copyright 2015 François-Xavier Bourlet .' From 20f71e0f4e67546cc304d7616545a064f5d6caac Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 11 Feb 2020 21:42:39 +1100 Subject: [PATCH 106/113] Fix simple typo: sream -> stream Closes #228 --- doc/protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/protocol.md b/doc/protocol.md index a5ee1ff..37a6417 100644 --- a/doc/protocol.md +++ b/doc/protocol.md @@ -231,7 +231,7 @@ FIXME we should rather standardize about the basic introspection calls. At the protocol level, streaming is straightforward. When a server wants to stream some data, instead of sending a "OK" message, it sends a "STREAM" message. The client will know that it needs to keep waiting for more. -At the end of the sream, a "STREAM_DONE" message is expected. +At the end of the stream, a "STREAM_DONE" message is expected. Formal definitions follow. From 23c2e28a801ab82a8dbc33a30bcc4cc6f01a3638 Mon Sep 17 00:00:00 2001 From: Shiplu Mokaddim Date: Wed, 20 May 2020 13:53:51 +0200 Subject: [PATCH 107/113] Using trusty as distro for python 2.6 build Travis changed the default distro to Xenial which doesn't have python 2.6 environment. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4a59b0a..267e190 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: trusty sudo: false language: python python: From 6237243ff24c1f1046115ac209f601ae698fcb66 Mon Sep 17 00:00:00 2001 From: Shiplu Mokaddim Date: Sun, 24 May 2020 09:35:55 +0200 Subject: [PATCH 108/113] add pip freeze in travis.yml To see the installed package versions --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 267e190..c01068d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ install: fi - "pip install nose $PYZMQ" - pip install . + - pip freeze script: - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then flake8 --ignore=E501,E128 zerorpc bin; From c43ac74a207ecf5e8e3b38621d15a6f2ec735581 Mon Sep 17 00:00:00 2001 From: Cedric Hombourger Date: Wed, 5 Jan 2022 21:23:52 +0100 Subject: [PATCH 109/113] try collections.abc instead of collections for Python >= 3.4 Kill a depreciation warning by importing "collections.abc" and fallback to "collections" on failure in order to maintain support for older versions of Python. Fixes: #231 Signed-off-by: Cedric Hombourger --- tests/test_reqstream.py | 10 ++++++++-- zerorpc/cli.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index 5d48b4d..71e1511 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -28,11 +28,17 @@ from builtins import range import gevent -import collections import zerorpc from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR +try: + # Try collections.abc for 3.4+ + from collections.abc import Iterator +except ImportError: + # Fallback to collections for Python 2 + from collections import Iterator + def test_rcp_streaming(): endpoint = random_ipc_endpoint() @@ -58,7 +64,7 @@ def xrange(self, max): assert list(r) == list(range(10)) r = client.xrange(10) - assert isinstance(r, collections.Iterator) + assert isinstance(r, Iterator) l = [] print('wait 4s for fun') gevent.sleep(TIME_FACTOR * 4) diff --git a/zerorpc/cli.py b/zerorpc/cli.py index 2907c25..3fd6350 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -33,11 +33,17 @@ import inspect import os import logging -import collections from pprint import pprint import zerorpc +try: + # Try collections.abc for 3.4+ + from collections.abc import Iterator +except ImportError: + # Fallback to collections for Python 2 + from collections import Iterator + parser = argparse.ArgumentParser( description='Make a zerorpc call to a remote service.' @@ -267,7 +273,7 @@ def run_client(args): else: call_args = args.params results = client(args.command, *call_args) - if not isinstance(results, collections.Iterator): + if not isinstance(results, Iterator): if args.print_json: json.dump(results, sys.stdout) else: From d6346f5687d235acdd2f4eec10acc7cfeb8ba572 Mon Sep 17 00:00:00 2001 From: Cedric Hombourger Date: Wed, 5 Jan 2022 21:05:09 +0100 Subject: [PATCH 110/113] core: handle both "async" and "async_" in remote calls Python 3.7 made "async" a reserved keyword: this causes a syntax error when passing "async" as a named parameter in remote procedure calls. Make ClientBase.__call__() understand both "async" and "async_" so we don't break our binary interface with older clients. Fixes: #239 Signed-off-by: Cedric Hombourger --- tests/test_client_async.py | 6 +++--- zerorpc/core.py | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_client_async.py b/tests/test_client_async.py index 3acbd3b..6ce4def 100644 --- a/tests/test_client_async.py +++ b/tests/test_client_async.py @@ -53,7 +53,7 @@ def add(self, a, b): client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - async_result = client.add(1, 4, async=True) + async_result = client.add(1, 4, async_=True) if sys.version_info < (2, 7): def _do_with_assert_raises(): @@ -84,8 +84,8 @@ def add(self, a, b): client = zerorpc.Client() client.connect(endpoint) - async_result = client.lolita(async=True) + async_result = client.lolita(async_=True) assert async_result.get() == 42 - async_result = client.add(1, 4, async=True) + async_result = client.add(1, 4, async_=True) assert async_result.get() == 5 diff --git a/zerorpc/core.py b/zerorpc/core.py index 9fa63e3..9dbf5cc 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -266,7 +266,10 @@ def __call__(self, method, *args, **kargs): self._context.hook_client_before_request(request_event) bufchan.emit_event(request_event) - if kargs.get('async', False) is False: + # In python 3.7, "async" is a reserved keyword, clients should now use + # "async_": support both for the time being + if (kargs.get('async', False) is False and + kargs.get('async_', False) is False): return self._process_response(request_event, bufchan, timeout) async_result = gevent.event.AsyncResult() From f662983af608db2da39773718ae9ac13ae266e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20K=C5=82oczko?= Date: Thu, 6 Jan 2022 02:40:22 +0000 Subject: [PATCH 111/113] Move from deprecated nose to pytest (solves #243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomasz Kłoczko --- .travis.yml | 2 +- setup.py | 4 ++-- tests/test_buffered_channel.py | 42 +++++++++++++++++----------------- tests/test_client_async.py | 6 ++--- tests/test_heartbeat.py | 34 +++++++++++++-------------- tests/test_middleware.py | 6 ++--- tests/test_server.py | 14 ++++++------ tests/testutils.py | 4 ++-- tox.ini | 4 ++-- 9 files changed, 58 insertions(+), 58 deletions(-) diff --git a/.travis.yml b/.travis.yml index c01068d..67c2d20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,4 +29,4 @@ script: - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then flake8 --ignore=E501,E128 zerorpc bin; fi - - nosetests -v + - pytest -v diff --git a/setup.py b/setup.py index 610e68b..b07ebcb 100644 --- a/setup.py +++ b/setup.py @@ -64,8 +64,8 @@ url='https://github.com/0rpc/zerorpc-python', packages=['zerorpc'], install_requires=requirements, - tests_require=['nose'], - test_suite='nose.collector', + tests_require=['pytest'], + test_suite='pytest.collector', zip_safe=False, entry_points={'console_scripts': ['zerorpc = zerorpc.cli:main']}, license='MIT', diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index 76d0fe3..20b8173 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -27,7 +27,7 @@ from __future__ import absolute_import from builtins import range -from nose.tools import assert_raises +import pytest import gevent import sys @@ -61,9 +61,9 @@ def test_close_server_bufchan(): print('CLOSE SERVER SOCKET!!!') server_bufchan.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_bufchan.recv) + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): client_bufchan.recv() print('CLIENT LOST SERVER :)') client_bufchan.close() @@ -96,9 +96,9 @@ def test_close_client_bufchan(): print('CLOSE CLIENT SOCKET!!!') client_bufchan.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_bufchan.recv) + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): client_bufchan.recv() print('SERVER LOST CLIENT :)') server_bufchan.close() @@ -129,9 +129,9 @@ def test_heartbeat_can_open_channel_server_close(): print('CLOSE SERVER SOCKET!!!') server_bufchan.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_bufchan.recv) + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): client_bufchan.recv() print('CLIENT LOST SERVER :)') client_bufchan.close() @@ -170,9 +170,9 @@ def server_fn(): client_bufchan.close() client.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, server_coro.get) + pytest.raises(zerorpc.LostRemote, server_coro.get) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): server_coro.get() print('SERVER LOST CLIENT :)') server.close() @@ -244,9 +244,9 @@ def client_do(): assert list(event.args) == [x + x * x] client_bufchan.emit('add', (x, x * x)) if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_bufchan.recv) + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): client_bufchan.recv() client_bufchan.close() @@ -308,9 +308,9 @@ def server_do(): server_bufchan.emit('OK', (sum(event.args),)) if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, server_bufchan.recv) + pytest.raises(zerorpc.LostRemote, server_bufchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): server_bufchan.recv() server_bufchan.close() @@ -343,9 +343,9 @@ def _do_with_assert_raises(): event = client_bufchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' assert list(event.args) == [x] - assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: - with assert_raises(zerorpc.TimeoutExpired): + with pytest.raises(zerorpc.TimeoutExpired): for x in range(10): client_bufchan.emit('sleep', (x,)) event = client_bufchan.recv(timeout=TIME_FACTOR * 3) @@ -369,9 +369,9 @@ def _do_with_assert_raises(): assert event.name == 'sleep' gevent.sleep(TIME_FACTOR * event.args[0]) server_bufchan.emit('OK', event.args) - assert_raises(zerorpc.LostRemote, _do_with_assert_raises) + pytest.raises(zerorpc.LostRemote, _do_with_assert_raises) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): for x in range(20): event = server_bufchan.recv() assert event.name == 'sleep' @@ -422,9 +422,9 @@ def server_do(): def _do_with_assert_raises(): for x in range(200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 - assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: - with assert_raises(zerorpc.TimeoutExpired): + with pytest.raises(zerorpc.TimeoutExpired): for x in range(200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 server_bufchan.emit('coucou', 1) # block until receiver is ready @@ -432,9 +432,9 @@ def _do_with_assert_raises(): def _do_with_assert_raises(): for x in range(2, 200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 - assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: - with assert_raises(zerorpc.TimeoutExpired): + with pytest.raises(zerorpc.TimeoutExpired): for x in range(2, 200): server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 for x in range(read_cnt.value, 200): diff --git a/tests/test_client_async.py b/tests/test_client_async.py index 3acbd3b..7d39364 100644 --- a/tests/test_client_async.py +++ b/tests/test_client_async.py @@ -25,7 +25,7 @@ from __future__ import print_function from __future__ import absolute_import -from nose.tools import assert_raises +import pytest import gevent import sys @@ -58,9 +58,9 @@ def add(self, a, b): if sys.version_info < (2, 7): def _do_with_assert_raises(): print(async_result.get()) - assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: - with assert_raises(zerorpc.TimeoutExpired): + with pytest.raises(zerorpc.TimeoutExpired): print(async_result.get()) client.close() srv.close() diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 75b1d29..14c66fd 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -27,7 +27,7 @@ from __future__ import absolute_import from builtins import range -from nose.tools import assert_raises +import pytest import gevent import sys @@ -59,9 +59,9 @@ def test_close_server_hbchan(): print('CLOSE SERVER SOCKET!!!') server_hbchan.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_hbchan.recv) + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): client_hbchan.recv() print('CLIENT LOST SERVER :)') client_hbchan.close() @@ -92,9 +92,9 @@ def test_close_client_hbchan(): print('CLOSE CLIENT SOCKET!!!') client_hbchan.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, server_hbchan.recv) + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): server_hbchan.recv() print('SERVER LOST CLIENT :)') server_hbchan.close() @@ -123,9 +123,9 @@ def test_heartbeat_can_open_channel_server_close(): print('CLOSE SERVER SOCKET!!!') server_hbchan.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_hbchan.recv) + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): client_hbchan.recv() print('CLIENT LOST SERVER :)') client_hbchan.close() @@ -155,9 +155,9 @@ def test_heartbeat_can_open_channel_client_close(): client_hbchan.close() client.close() if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, server_hbchan.recv) + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): server_hbchan.recv() print('SERVER LOST CLIENT :)') server_hbchan.close() @@ -227,9 +227,9 @@ def client_do(): assert list(event.args) == [x + x * x] client_hbchan.emit('add', (x, x * x)) if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, client_hbchan.recv) + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): client_hbchan.recv() client_hbchan.close() @@ -287,9 +287,9 @@ def server_do(): server_hbchan.emit('OK', (sum(event.args),)) if sys.version_info < (2, 7): - assert_raises(zerorpc.LostRemote, server_hbchan.recv) + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): server_hbchan.recv() server_hbchan.close() @@ -322,9 +322,9 @@ def _do_with_assert_raises(): event = client_hbchan.recv(timeout=TIME_FACTOR * 3) assert event.name == 'OK' assert list(event.args) == [x] - assert_raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) else: - with assert_raises(zerorpc.TimeoutExpired): + with pytest.raises(zerorpc.TimeoutExpired): for x in range(10): client_hbchan.emit('sleep', (x,)) event = client_hbchan.recv(timeout=TIME_FACTOR * 3) @@ -346,9 +346,9 @@ def _do_with_assert_raises(): assert event.name == 'sleep' gevent.sleep(TIME_FACTOR * event.args[0]) server_hbchan.emit('OK', event.args) - assert_raises(zerorpc.LostRemote, _do_with_assert_raises) + pytest.raises(zerorpc.LostRemote, _do_with_assert_raises) else: - with assert_raises(zerorpc.LostRemote): + with pytest.raises(zerorpc.LostRemote): for x in range(20): event = server_hbchan.recv() assert event.name == 'sleep' diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 754f8cb..3163a3a 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -28,7 +28,7 @@ from builtins import str from future.utils import tobytes -from nose.tools import assert_raises +import pytest import gevent import gevent.local import random @@ -101,9 +101,9 @@ def hello(self): srv = Srv(heartbeat=TIME_FACTOR * 1, context=c) if sys.version_info < (2, 7): - assert_raises(zmq.ZMQError, srv.bind, 'some_service') + pytest.raises(zmq.ZMQError, srv.bind, 'some_service') else: - with assert_raises(zmq.ZMQError): + with pytest.raises(zmq.ZMQError): srv.bind('some_service') cnt = c.register_middleware(Resolver()) diff --git a/tests/test_server.py b/tests/test_server.py index 2a266ce..86997a9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -27,7 +27,7 @@ from __future__ import absolute_import from builtins import range -from nose.tools import assert_raises +import pytest import gevent import sys @@ -114,9 +114,9 @@ def add(self, a, b): client.connect(endpoint) if sys.version_info < (2, 7): - assert_raises(zerorpc.TimeoutExpired, client.add, 1, 4) + pytest.raises(zerorpc.TimeoutExpired, client.add, 1, 4) else: - with assert_raises(zerorpc.TimeoutExpired): + with pytest.raises(zerorpc.TimeoutExpired): print(client.add(1, 4)) client.close() srv.close() @@ -140,9 +140,9 @@ def raise_something(self, a): if sys.version_info < (2, 7): def _do_with_assert_raises(): print(client.raise_something(42)) - assert_raises(zerorpc.RemoteError, _do_with_assert_raises) + pytest.raises(zerorpc.RemoteError, _do_with_assert_raises) else: - with assert_raises(zerorpc.RemoteError): + with pytest.raises(zerorpc.RemoteError): print(client.raise_something(42)) assert client.raise_something(list(range(5))) == 4 client.close() @@ -167,9 +167,9 @@ def raise_error(self): if sys.version_info < (2, 7): def _do_with_assert_raises(): print(client.raise_error()) - assert_raises(zerorpc.RemoteError, _do_with_assert_raises) + pytest.raises(zerorpc.RemoteError, _do_with_assert_raises) else: - with assert_raises(zerorpc.RemoteError): + with pytest.raises(zerorpc.RemoteError): print(client.raise_error()) try: client.raise_error() diff --git a/tests/testutils.py b/tests/testutils.py index 2a0110c..85b6a96 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -26,7 +26,7 @@ from builtins import str import functools -import nose.exc +import pytest import random import os @@ -52,7 +52,7 @@ def skip(reason): def _skip(test): @functools.wraps(test) def wrap(): - raise nose.exc.SkipTest(reason) + raise pytest.SkipTest(reason) return wrap return _skip diff --git a/tox.ini b/tox.ini index 3490b2c..96bace8 100644 --- a/tox.ini +++ b/tox.ini @@ -4,10 +4,10 @@ envlist = py26,py27,py34,py35 [testenv] deps = flake8 - nose + pytest commands = flake8 zerorpc bin - nosetests -v + pytest -v passenv = ZPC_TEST_TIME_FACTOR [flake8] From dd6843c114e9bfd6e81abf68f6a2eb3c598d1c2f Mon Sep 17 00:00:00 2001 From: Cedric Hombourger Date: Thu, 19 May 2022 22:54:27 +0200 Subject: [PATCH 112/113] gevent_zmq: import enums from pyzmq >= 23.0.0 With pyzmq 23.0.0, constants were changed to enums and moved to the constants module. Attempt to import all globals from it into our zmq wrapper. Closes: #251 Signed-off-by: Cedric Hombourger --- zerorpc/gevent_zmq.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index 9430695..54420ae 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -27,6 +27,13 @@ # We want to act like zmq from zmq import * # noqa +try: + # Try to import enums for pyzmq >= 23.0.0 + from zmq.constants import * # noqa +except ImportError: + pass + + # Explicit import to please flake8 from zmq import ZMQError From be578354c09c43412d052c53de14745868bd0c4f Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Wed, 20 Jul 2022 18:20:11 +1000 Subject: [PATCH 113/113] docs: Fix a few typos There are small typos in: - doc/protocol.md - zerorpc/core.py Fixes: - Should read `transferred` rather than `transfered`. - Should read `occurred` rather than `occured`. Signed-off-by: Tim Gates --- doc/protocol.md | 2 +- zerorpc/core.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/protocol.md b/doc/protocol.md index 37a6417..cfc83d2 100644 --- a/doc/protocol.md +++ b/doc/protocol.md @@ -207,7 +207,7 @@ exception is raised), we use the ERR event. - Name of the error (it should be the exception class name, or another meaningful keyword). - Human representation of the error (preferably in english). - - If possible a pretty printed traceback of the call stack when the error occured. + - If possible a pretty printed traceback of the call stack when the error occurred. > A future version of the protocol will probably add a structured version of the > traceback, allowing machine-to-machine stack walking and better cross-language diff --git a/zerorpc/core.py b/zerorpc/core.py index 9dbf5cc..ec2e008 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -406,7 +406,7 @@ def fork_task_context(functor, context=None): - task1 is created to handle this event this task will be linked to the initial event context. zerorpc.Server does that for you. - task1 make use of some zerorpc.Client instances, the initial - event context is transfered on every call. + event context is transferred on every call. - task1 spawn a new task2. - task2 make use of some zerorpc.Client instances, it's a fresh @@ -415,7 +415,7 @@ def fork_task_context(functor, context=None): - task1 spawn a new fork_task_context(task3). - task3 make use of some zerorpc.Client instances, the initial - event context is transfered on every call. + event context is transferred on every call. A real use case is a distributed tracer. Each time a new event is created, a trace_id is injected in it or copied from the current task