diff --git a/.gitignore b/.gitignore index 58311d7..8aa21a8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ *.egg *.egg-info .tox +.idea diff --git a/.travis.yml b/.travis.yml index 2a1a60c..67c2d20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,32 @@ +dist: trusty +sudo: false language: python python: + - 2.6 - 2.7 -script: nosetests -before_install: - - sudo apt-get update - - sudo apt-get install python-dev libevent-dev + - 3.4 + - 3.5 +env: + matrix: + - PYZMQ='pyzmq>=15' + - PYZMQ='pyzmq>=14,<15' + - PYZMQ='pyzmq>=13,<14' +matrix: + fast_finish: true +addons: + apt: + packages: + - python-dev + - libevent-dev install: - - pip install nose --use-mirrors - - pip install . --use-mirrors + - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then + pip install flake8; + fi + - "pip install nose $PYZMQ" + - pip install . + - pip freeze +script: + - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then + flake8 --ignore=E501,E128 zerorpc bin; + fi + - pytest -v diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 6e329ee..0000000 --- a/AUTHORS +++ /dev/null @@ -1,6 +0,0 @@ -Andrea Luzzardi -François-Xavier Bourlet -Jérôme Petazzoni -Samuel Alba -Solomon Hykes -Sébastien Pahl diff --git a/LICENSE b/LICENSE index 43fb7a1..49bbb6e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ Open Source Initiative OSI - The MIT License (MIT):Licensing The MIT License (MIT) -Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.rst b/README.rst index 481a513..3538015 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ zerorpc ======= -.. image:: https://secure.travis-ci.org/dotcloud/zerorpc-python.png - :target: http://travis-ci.org/dotcloud/zerorpc-python +.. image:: https://travis-ci.org/0rpc/zerorpc-python.svg?branch=master + :target: https://travis-ci.org/0rpc/zerorpc-python Mailing list: zerorpc@googlegroups.com (https://groups.google.com/d/forum/zerorpc) @@ -16,6 +16,14 @@ with a convenient script, "zerorpc", allowing to: * expose Python modules without modifying a single line of code, * call those modules remotely through the command line. +Installation +------------ + +On most systems, its a matter of:: + + $ pip install zerorpc + +Depending of the support from Gevent and PyZMQ on your system, you might need to install `libev` (for gevent) and `libzmq` (for pyzmq) with the development files. Create a server with a one-liner -------------------------------- @@ -28,7 +36,7 @@ we will expose the Python "time" module:: .. note:: The bind address uses the zeromq address format. You are not limited to TCP transport: you could as well specify ipc:///tmp/time to use - host-local sockets, for instance. "tcp://*:1234" is a short-hand to + host-local sockets, for instance. "tcp://\*:1234" is a short-hand to "tcp://0.0.0.0:1234" and means "listen on TCP port 1234, accepting connections on all IP addresses". @@ -108,15 +116,15 @@ your server to act as a kind of worker, and connect to a hub or queue which will dispatch requests. You can achieve this by swapping "--bind" and "--connect":: - $ zerorpc --bind tcp://*:1234 localtime + $ zerorpc --bind tcp://*:1234 strftime %Y/%m/%d -We now have "something" wanting to call the "localtime" function, and waiting +We now have "something" wanting to call the "strftime" function, and waiting for a worker to connect to it. Let's start the worker:: - $ zerorpc --server tcp://*:1234 time + $ zerorpc --server tcp://127.0.0.1:1234 time The worker will connect to the listening client and ask him "what should I -do?"; the client will send the "localtime" function call; the worker will +do?"; the client will send the "strftime" function call; the worker will execute it and return the result. The first program will display the local time and exit. The worker will remain running. @@ -129,7 +137,7 @@ the "--bind" option:: $ zerorpc --server --bind tcp://*:1234 --bind ipc:///tmp/time time -You can then connect to it using either "zerorpc tcp://*:1234" or +You can then connect to it using either "zerorpc tcp://\*:1234" or "zerorpc ipc:///tmp/time". Wait, there is more! You can even mix "--bind" and "--connect". That means @@ -156,7 +164,7 @@ python API. Below are a few examples. Here's how to expose an object of your choice as a zeroservice:: - class Cooler: + class Cooler(object): """ Various convenience methods to make things cooler. """ def add_man(self, sentence): @@ -186,11 +194,11 @@ Let's save this code to *cooler.py* and run it:: Now, in another terminal, let's try connecting to our awesome zeroservice:: - $ zerorpc -j tcp://:4242 add_42 1 + $ zerorpc -j tcp://localhost:4242 add_42 1 43 - $ zerorpc tcp://:4242 add_man 'I own a mint-condition Wolkswagen Golf' - "I own a mint-condition Wolkswagen Gold, man!" - $ zerorpc tcp://:4242 boat 'I own a mint-condition Wolkswagen Gold, man!' + $ zerorpc tcp://localhost:4242 add_man 'I own a mint-condition Volkswagen Golf' + "I own a mint-condition Volkswagen Golf, man!" + $ zerorpc tcp://localhost:4242 boat 'I own a mint-condition Volkswagen Golf, man!' "I'm on a boat!" diff --git a/bin/zerorpc b/bin/zerorpc index 41ab2dc..c9d31d3 100755 --- a/bin/zerorpc +++ b/bin/zerorpc @@ -3,7 +3,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -27,7 +27,7 @@ import os import sys sys.path.append(os.path.dirname(os.path.dirname(sys.argv[0]))) -from zerorpc import cli +from zerorpc import cli # NOQA if __name__ == "__main__": exit(cli.main()) diff --git a/doc/protocol.md b/doc/protocol.md index 6bd2ad7..cfc83d2 100644 --- a/doc/protocol.md +++ b/doc/protocol.md @@ -1,25 +1,51 @@ -# ZeroRPC Protocol +# zerorpc Protocol THIS DOCUMENT IS INCOMPLETE, WORK IN PROGRESS! -This document attempts to define the ZeroRPC protocol. +This document attempts to define the zerorpc protocol. ## Introduction & History -[ZeroRPC](http://zerorpc.dotcloud.com) is a modern communication layer for distributed systems built on top of [ZeroMQ](http://zeromq.org), developed at [dotCloud](http://dotcloud.com) since 2010 and open-sourced in 2012. It features a dead-simple API for exposing any object or module over the network, and a powerful gevent implementation which supports multiple ZMQ socket types, streaming, heartbeats and more. It also includes a simple command-line tool for interactive querying and introspection. The platform team at dotCloud uses it in production to transmit millions of messages per day across hundreds of services. +zerorpc features a dead-simple API for exposing any object or module over the +network, and a powerful gevent implementation which supports multiple ZMQ +socket types, streaming, heartbeats and more. It also includes a simple +command-line tool for interactive querying and introspection. -ZeroRPC uses ZMQ as a transport, but uses a communication protocol that is transport-agnostic. For a long time the reference documentation for that protocol was the python code itself. However since its recent surge in popularity many people have expressed interest in porting it to other programming languages. We hope that this standalone protocol documentation will help them. - -A short warning: ZeroRPC started as a simple tool to solve a simple problem. It was progressively refined and improved to satisfy the growing needs of the dotCloud platform. The emphasis is on practicality, robustness and operational simplicity - sometimes at the expense of formal elegance and optimizations. We will gladly welcome patches focused on the latter so long as they don't hurt the former. +zerorpc uses ZMQ as a transport, but uses a communication protocol that is +transport-agnostic. For a long time the reference documentation for that +protocol was the python code itself. However since its recent surge in +popularity many people have expressed interest in porting it to other +programming languages. We hope that this standalone protocol documentation will +help them. > The python implementation of zerorpc act as a reference for the whole > protocol. New features and experiments are implemented and tested in this -> version first. This is also this implementation that is powering dotCloud's -> infrastructure. +> version first. + +[zerorpc](http://www.zerorpc.io) is a modern communication layer for +distributed systems built on top of [ZeroMQ](http://zeromq.org), initially +developed at [dotCloud](http://www.dotcloud.com) starting in 2010 and +open-sourced in 2012. when dotCloud pivoted to [Docker](http://www.docker.com), +dotCloud was acquired by [cloudControl](https://www.cloudcontrol.com/), which +then migrated over to theirs own PaaS before shutting it down. + +In 2015, I (François-Xavier Bourlet) was given zerorpc by cloudControl, in an +attempt to revive the project, maintain it, and hopefully drive its +development. + +### Warning + +A short warning: zerorpc started as a simple tool to solve a simple problem. It +was progressively refined and improved to satisfy the growing needs of the +dotCloud platform. The emphasis is on practicality, robustness and operational +simplicity - sometimes at the expense of formal elegance and optimizations. We +will gladly welcome patches focused on the latter so long as they don't hurt +the former. + ## Layers -Before diving into any details, let's divide ZeroRPC's protocol in three +Before diving into any details, let's divide zerorpc's protocol in three different layers: 1. Wire (or transport) layer; a combination of ZMQ @@ -34,15 +60,15 @@ The wire layer is a combination of ZMQ and msgpack. The basics: - - A ZeroRPC server can listen on as many ZMQ sockets as you like. Actually, + - A zerorpc server can listen on as many ZMQ sockets as you like. Actually, a ZMQ socket can bind to multiple addresses. It can even *connect* to the clients (think about it as a worker connecting to a hub), but there are - some limitations in that case (see below). ZeroRPC doesn't + some limitations in that case (see below). zerorpc doesn't have to do anything specific for that: ZMQ handles it automatically. - - A ZeroRPC client can connect to multiple ZeroRPC servers. However, it should + - A zerorpc client can connect to multiple zerorpc servers. However, it should create a new ZMQ socket for each connection. -Since ZeroRPC implements heartbeat and streaming, it expects a kind of +Since zerorpc implements heartbeat and streaming, it expects a kind of persistent, end-to-end, connection between the client and the server. It means that we cannot use the load-balancing features built into ZMQ. Otherwise, the various messages composing a single conversation could @@ -52,12 +78,12 @@ That's why there are limitations when the server connects to the client: if there are multiple servers connecting to the same client, bad things will happen. -> Note that the current implementation of ZeroRPC for Python doesn't implement +> Note that the current implementation of zerorpc for Python doesn't implement > its own load-balancing (yet), and still uses one ZMQ socket for connecting to > many servers. You can still use ZMQ load-balancing if you accept to disable > heartbeat and don't use streamed responses. -Every event from the event layer will be serialized with msgpack. +Every event from the event layer will be serialized with msgpack. ## Event layer @@ -103,12 +129,12 @@ This document talks only about the version 3 of the protocol. - Each new event opens a new channel implicitly. - The id of the new event will represent the channel id for the connection. - - Each consecutive event on a channel will have the header field "reply_to" + - Each consecutive event on a channel will have the header field "response_to" set to the channel id: { "message_id": "6ce9503a-bfb8-486a-ac79-e2ed225ace79", - "reply_to": "6636fb60-2bca-4ecb-8cb4-bbaaf174e9e6" + "response_to": "6636fb60-2bca-4ecb-8cb4-bbaaf174e9e6" } #### Heartbeat @@ -140,18 +166,18 @@ size of its local buffer. This is a hint for the remote, to tell it "send me more data!" - Event's name: '\_zpc\_more' - - Event's args: integer representing how many entries are available in the client's buffer. + - Event's args: integer representing how many entries are available in the client's buffer. FIXME WIP ## RPC Layer -In the first version of ZeroRPC, this was the main (and only) layer. +In the first version of zerorpc, this was the main (and only) layer. Three kinds of events can occur at this layer: request (=function call), -response (=function return), error (=exception). +response (=function return), error (=exception). Request: - + - Event's name: string with the name of the method to call. - Event's args: tuple of arguments for the method. @@ -160,14 +186,14 @@ support them. If you absolutely want to call functions with keyword arguments, you can use a wrapper; e.g. expose a function like "call_with_kwargs(function_name, args, kwargs)", where args is a list, and kwargs a dict. It might be an interesting idea to add such a -helper function into ZeroRPC default methods (see below for definitions +helper function into zerorpc default methods (see below for definitions of existing default methods). Response: - Event's name: string "OK" - Event's args: tuple containing the returned value - + > Note that if the return value is a tuple, it is itself wrapped inside a > tuple - again, for backward compatibility reasons. @@ -178,10 +204,10 @@ exception is raised), we use the ERR event. - Event's name: string "ERR" - Event's args: tuple of 3 strings: - - Name of the error (it should be the exception class name, or another + - Name of the error (it should be the exception class name, or another meaningful keyword). - Human representation of the error (preferably in english). - - If possible a pretty printed traceback of the call stack when the error occured. + - If possible a pretty printed traceback of the call stack when the error occurred. > A future version of the protocol will probably add a structured version of the > traceback, allowing machine-to-machine stack walking and better cross-language @@ -190,7 +216,7 @@ exception is raised), we use the ERR event. ### Default calls -When exposing some code with ZeroRPC, a number of methods/functions are +When exposing some code with zerorpc, a number of methods/functions are automatically added, to provide useful debugging and development tools. - \_zerorpc\_ping() just answers with a pong message. @@ -205,7 +231,7 @@ FIXME we should rather standardize about the basic introspection calls. At the protocol level, streaming is straightforward. When a server wants to stream some data, instead of sending a "OK" message, it sends a "STREAM" message. The client will know that it needs to keep waiting for more. -At the end of the sream, a "STREAM_DONE" message is expected. +At the end of the stream, a "STREAM_DONE" message is expected. Formal definitions follow. @@ -213,7 +239,7 @@ Messages part of a stream: - Event's name: string "STREAM" - Event's args: tuple containing the streamed value - + When the STREAM reaches its end: - Event's name: string "STREAM\_DONE" diff --git a/setup.py b/setup.py index c4acc49..b07ebcb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,9 +22,12 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -execfile('zerorpc/version.py') import sys +if sys.version_info < (3, 0): + execfile('zerorpc/version.py') +else: + exec(compile(open('zerorpc/version.py', encoding='utf8').read(), 'zerorpc/version.py', 'exec')) try: from setuptools import setup @@ -33,24 +36,36 @@ requirements = [ - 'gevent', - 'msgpack-python', - 'pyzmq>=2.2.0.1' + 'msgpack>=0.5.2', + 'pyzmq>=13.1.0', + 'future', ] + if sys.version_info < (2, 7): requirements.append('argparse') +if sys.version_info < (2, 7): + requirements.append('gevent>=1.1.0,<1.2.0') +elif sys.version_info < (3, 0): + requirements.append('gevent>=1.0') +else: + requirements.append('gevent>=1.1') + +with open("README.rst", "r") as fh: + long_description = fh.read() setup( name='zerorpc', version=__version__, description='zerorpc is a flexible RPC based on zeromq.', + long_description=long_description, + long_description_content_type='text/x-rst', author=__author__, - url='https://github.com/dotcloud/zerorpc-python', + url='https://github.com/0rpc/zerorpc-python', packages=['zerorpc'], install_requires=requirements, - tests_require=['nose'], - test_suite='nose.collector', + tests_require=['pytest'], + test_suite='pytest.collector', zip_safe=False, entry_points={'console_scripts': ['zerorpc = zerorpc.cli:main']}, license='MIT', @@ -60,5 +75,8 @@ 'Natural Language :: English', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', ), ) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index b223640..20b8173 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,41 +23,49 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + +import pytest import gevent +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_bufchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_bufchan.close() - with assert_raises(zerorpc.LostRemote): - client_bufchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() + print('CLIENT LOST SERVER :)') client_bufchan.close() server.close() client.close() @@ -65,31 +73,34 @@ def test_close_server_bufchan(): def test_close_client_bufchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_bufchan.close() - with assert_raises(zerorpc.LostRemote): - server_bufchan.recv() - print 'SERVER LOST CLIENT :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() + print('SERVER LOST CLIENT :)') server_bufchan.close() server.close() client.close() @@ -97,29 +108,32 @@ def test_close_client_bufchan(): def test_heartbeat_can_open_channel_server_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_bufchan.close() - with assert_raises(zerorpc.LostRemote): - client_bufchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() + print('CLIENT LOST SERVER :)') client_bufchan.close() server.close() client.close() @@ -127,268 +141,369 @@ def test_heartbeat_can_open_channel_server_close(): def test_heartbeat_can_open_channel_client_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) - - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + def server_fn(): + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan) + try: + while True: + gevent.sleep(1) + finally: + server_bufchan.close() + server_coro = gevent.spawn(server_fn) + + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_bufchan.close() client.close() - with assert_raises(zerorpc.LostRemote): - server_bufchan.recv() - print 'SERVER LOST CLIENT :)' - server_bufchan.close() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_coro.get) + else: + with pytest.raises(zerorpc.LostRemote): + server_coro.get() + print('SERVER LOST CLIENT :)') server.close() def test_do_some_req_rep(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - client_bufchan = zerorpc.BufferedChannel(client_hbchan) - - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) def client_do(): - for x in xrange(20): + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan) + for x in range(20): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): - for x in xrange(20): + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan) + + for x in range(20): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() def test_do_some_req_rep_lost_server(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): - print 'running' + print('running') client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - for x in xrange(10): + for x in range(10): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_bufchan.emit('add', (x, x * x)) - with assert_raises(zerorpc.LostRemote): - event = client_bufchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - for x in xrange(10): + for x in range(10): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() def test_do_some_req_rep_lost_client(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - for x in xrange(10): + for x in range(10): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - for x in xrange(10): + for x in range(10): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) - with assert_raises(zerorpc.LostRemote): - event = server_bufchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_bufchan.recv() server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() def test_do_some_req_rep_client_timeout(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): - client_bufchan.emit('sleep', (x,)) - event = client_bufchan.recv(timeout=3) - assert event.name == 'OK' - assert event.args == (x,) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(10): + client_bufchan.emit('sleep', (x,)) + event = client_bufchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(10): + client_bufchan.emit('sleep', (x,)) + event = client_bufchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - with assert_raises(zerorpc.LostRemote): - for x in xrange(20): - event = server_bufchan.recv() - assert event.name == 'sleep' - gevent.sleep(event.args[0]) - server_bufchan.emit('OK', event.args) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(20): + event = server_bufchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_bufchan.emit('OK', event.args) + pytest.raises(zerorpc.LostRemote, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.LostRemote): + for x in range(20): + event = server_bufchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_bufchan.emit('OK', event.args) server_bufchan.close() - server_task = gevent.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.spawn(server_do) + + coro_pool.join() client.close() server.close() -class CongestionError(Exception): - pass - - def test_congestion_control_server_pushing(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - client_bufchan = zerorpc.BufferedChannel(client_hbchan) - - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) + read_cnt = type('Dummy', (object,), { "value": 0 }) def client_do(): - for x in xrange(200): + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=100) + for x in range(200): event = client_bufchan.recv() assert event.name == 'coucou' assert event.args == x + read_cnt.value += 1 + client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): - with assert_raises(CongestionError): - for x in xrange(200): - if server_bufchan.emit('coucou', x, block=False) == False: - raise CongestionError() # will fail when x == 1 + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=100) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 server_bufchan.emit('coucou', 1) # block until receiver is ready - with assert_raises(CongestionError): - for x in xrange(2, 200): - if server_bufchan.emit('coucou', x, block=False) == False: - raise CongestionError() # will fail when x == 100 - for x in xrange(101, 200): + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(2, 200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(2, 200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 + for x in range(read_cnt.value, 200): server_bufchan.emit('coucou', x) # block until receiver is ready + server_bufchan.close() + + coro_pool.spawn(server_do) + try: + coro_pool.join() + except zerorpc.LostRemote: + pass + finally: + client.close() + server.close() + + +def test_on_close_if(): + """ + Test that the on_close_if method does not cause exceptions when the client + is slow to recv() data. + """ + endpoint = random_ipc_endpoint() + server_events = zerorpc.Events(zmq.ROUTER) + server_events.bind(endpoint) + server = zerorpc.ChannelMultiplexer(server_events) + + client_events = zerorpc.Events(zmq.DEALER) + client_events.connect(endpoint) + client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) + + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=10) - server_task = gevent.spawn(server_do) + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=10) + + seen = [] + + def is_stream_done(event): + return event.name == 'done' + + def client_do(): + while True: + event = client_bufchan.recv() + if event.name == 'done': + return + seen.append(event.args) + gevent.sleep(0.1) + + def server_do(): + for i in range(0, 10): + server_bufchan.emit('blah', (i)) + server_bufchan.emit('done', ('bye')) + + client_bufchan.on_close_if = is_stream_done + + coro_pool = gevent.pool.Pool() + g1 = coro_pool.spawn(client_do) + g2 = coro_pool.spawn(server_do) + + g1.get() # Re-raise any exceptions... + g2.get() + + assert seen == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - server_task.get() - client_task.get() client_bufchan.close() - client.close() - server_task.get() server_bufchan.close() + client.close() server.close() diff --git a/tests/test_channel.py b/tests/test_channel.py index 4eebd65..1d59b1e 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,18 +23,22 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_events_channel_client_side(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -42,24 +46,25 @@ def test_events_channel_client_side(): client_channel.emit('someevent', (42,)) event = server.recv() - print event - assert event.args == (42,) - assert event.header.get('zmqid', None) is not None - - server.emit('someanswer', (21,), - xheader=dict(response_to=event.header['message_id'], - zmqid=event.header['zmqid'])) + print(event) + assert list(event.args) == [42] + assert event.identity is not None + + reply_event = server.new_event('someanswer', (21,), + xheader={'response_to': event.header['message_id']}) + reply_event.identity = event.identity + server.emit_event(reply_event) event = client_channel.recv() - assert event.args == (21,) + assert list(event.args) == [21] def test_events_channel_client_side_server_send_many(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -67,26 +72,27 @@ def test_events_channel_client_side_server_send_many(): client_channel.emit('giveme', (10,)) event = server.recv() - print event - assert event.args == (10,) - assert event.header.get('zmqid', None) is not None - - for x in xrange(10): - server.emit('someanswer', (x,), - xheader=dict(response_to=event.header['message_id'], - zmqid=event.header['zmqid'])) - for x in xrange(10): + print(event) + assert list(event.args) == [10] + assert event.identity is not None + + for x in range(10): + reply_event = server.new_event('someanswer', (x,), + xheader={'response_to': event.header['message_id']}) + reply_event.identity = event.identity + server.emit_event(reply_event) + for x in range(10): event = client_channel.recv() - assert event.args == (x,) + assert list(event.args) == [x] def test_events_channel_both_side(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -94,21 +100,21 @@ def test_events_channel_both_side(): client_channel.emit('openthat', (42,)) event = server.recv() - print event - assert event.args == (42,) + print(event) + assert list(event.args) == [42] assert event.name == 'openthat' server_channel = server.channel(event) server_channel.emit('test', (21,)) event = client_channel.recv() - assert event.args == (21,) + assert list(event.args) == [21] assert event.name == 'test' server_channel.emit('test', (22,)) event = client_channel.recv() - assert event.args == (22,) + assert list(event.args) == [22] assert event.name == 'test' server_events.close() diff --git a/tests/test_client.py b/tests/test_client.py index 599b0e1..6a692b3 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,10 +23,11 @@ # SOFTWARE. +from __future__ import absolute_import import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_client_connect(): endpoint = random_ipc_endpoint() diff --git a/tests/test_client_async.py b/tests/test_client_async.py index c939101..ced4b1f 100644 --- a/tests/test_client_async.py +++ b/tests/test_client_async.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2013 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,12 +23,15 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +import pytest import gevent +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_client_server_client_timeout_with_async(): @@ -40,19 +43,25 @@ def lolita(self): return 42 def add(self, a, b): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) return a + b srv = MySrv() srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - async_result = client.add(1, 4, async=True) - with assert_raises(zerorpc.TimeoutExpired): - print async_result.get() + async_result = client.add(1, 4, async_=True) + + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print(async_result.get()) + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + print(async_result.get()) client.close() srv.close() @@ -75,8 +84,8 @@ def add(self, a, b): client = zerorpc.Client() client.connect(endpoint) - async_result = client.lolita(async=True) + async_result = client.lolita(async_=True) assert async_result.get() == 42 - async_result = client.add(1, 4, async=True) + async_result = client.add(1, 4, async_=True) assert async_result.get() == 5 diff --git a/tests/test_client_heartbeat.py b/tests/test_client_heartbeat.py index e9de869..6b552a4 100644 --- a/tests/test_client_heartbeat.py +++ b/tests/test_client_heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,10 +23,15 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import next +from builtins import range + import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_client_server_hearbeat(): @@ -38,17 +43,17 @@ def lolita(self): return 42 def slow(self): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 1) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=1) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1) client.connect(endpoint) assert client.lolita() == 42 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_server_activate_heartbeat(): @@ -57,19 +62,19 @@ def test_client_server_activate_heartbeat(): class MySrv(zerorpc.Server): def lolita(self): - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) return 42 - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) gevent.sleep(0) - client = zerorpc.Client(heartbeat=1) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4) client.connect(endpoint) assert client.lolita() == 42 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_server_passive_hearbeat(): @@ -81,18 +86,19 @@ def lolita(self): return 42 def slow(self): - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) return 2 - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) + gevent.sleep(0) - client = zerorpc.Client(heartbeat=1, passive_heartbeat=True) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4, passive_heartbeat=True) client.connect(endpoint) assert client.slow() == 2 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_hb_doesnt_linger_on_streaming(): @@ -102,18 +108,18 @@ class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): - return xrange(42) + return range(42) - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - assert list(client1.iter()) == list(xrange(42)) - print 'sleep 3s' - gevent.sleep(3) + assert list(client1.iter()) == list(range(42)) + print('sleep 3s') + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -126,18 +132,18 @@ class MySrv(zerorpc.Server): def lolita(self): return 42 - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) - client2 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) - client3 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) + client2 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) + client3 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) assert client1.lolita() == 42 assert client2.lolita() == 42 - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) assert client3.lolita() == 42 @@ -150,18 +156,18 @@ class MySrv(zerorpc.Server): def iter(self): return [] - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - print 'grab iter' + print('grab iter') i = client1.iter() - print 'sleep 3s' - gevent.sleep(3) + print('sleep 3s') + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -173,22 +179,22 @@ class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): - return xrange(500) + return range(500) - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - print 'grab iter' + print('grab iter') i = client1.iter() - print 'consume some' - assert list(next(i) for x in xrange(142)) == list(xrange(142)) + print('consume some') + assert list(next(i) for x in range(142)) == list(range(142)) - print 'sleep 3s' - gevent.sleep(3) + print('sleep 3s') + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() diff --git a/tests/test_events.py b/tests/test_events.py index 3d4766a..7acf98e 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,11 +23,15 @@ # SOFTWARE. +from __future__ import print_function, absolute_import +from builtins import str, bytes +from builtins import range, object + from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint -class MokupContext(): +class MokupContext(object): _next_id = 0 def new_msgid(self): @@ -44,46 +48,46 @@ def test_context(): def test_event(): context = MokupContext() event = zerorpc.Event('mylittleevent', (None,), context=context) - print event + print(event) assert event.name == 'mylittleevent' assert event.header['message_id'] == 0 assert event.args == (None,) event = zerorpc.Event('mylittleevent2', ('42',), context=context) - print event + print(event) assert event.name == 'mylittleevent2' assert event.header['message_id'] == 1 assert event.args == ('42',) event = zerorpc.Event('mylittleevent3', ('a', 42), context=context) - print event + print(event) assert event.name == 'mylittleevent3' assert event.header['message_id'] == 2 assert event.args == ('a', 42) - event = zerorpc.Event('mylittleevent4', ('b', 21), context=context) - print event + event = zerorpc.Event('mylittleevent4', ('', 21), context=context) + print(event) assert event.name == 'mylittleevent4' assert event.header['message_id'] == 3 - assert event.args == ('b', 21) + assert event.args == ('', 21) packed = event.pack() unpacked = zerorpc.Event.unpack(packed) - print unpacked + print(unpacked) assert unpacked.name == 'mylittleevent4' assert unpacked.header['message_id'] == 3 - assert unpacked.args == ('b', 21) + assert list(unpacked.args) == ['', 21] event = zerorpc.Event('mylittleevent5', ('c', 24, True), header={'lol': 'rofl'}, context=None) - print event + print(event) assert event.name == 'mylittleevent5' assert event.header['lol'] == 'rofl' assert event.args == ('c', 24, True) event = zerorpc.Event('mod', (42,), context=context) - print event + print(event) assert event.name == 'mod' assert event.header['message_id'] == 4 assert event.args == (42,) @@ -102,9 +106,9 @@ def test_events_req_rep(): client.emit('myevent', ('arg1',)) event = server.recv() - print event + print(event) assert event.name == 'myevent' - assert event.args == ('arg1',) + assert list(event.args) == ['arg1'] def test_events_req_rep2(): @@ -115,41 +119,42 @@ def test_events_req_rep2(): client = zerorpc.Events(zmq.REQ) client.connect(endpoint) - for i in xrange(10): + for i in range(10): client.emit('myevent' + str(i), (i,)) event = server.recv() - print event + print(event) assert event.name == 'myevent' + str(i) - assert event.args == (i,) + assert list(event.args) == [i] server.emit('answser' + str(i * 2), (i * 2,)) event = client.recv() - print event + print(event) assert event.name == 'answser' + str(i * 2) - assert event.args == (i * 2,) + assert list(event.args) == [i * 2] def test_events_dealer_router(): endpoint = random_ipc_endpoint() - server = zerorpc.Events(zmq.XREP) + server = zerorpc.Events(zmq.ROUTER) server.bind(endpoint) - client = zerorpc.Events(zmq.XREQ) + client = zerorpc.Events(zmq.DEALER) client.connect(endpoint) - for i in xrange(6): + for i in range(6): client.emit('myevent' + str(i), (i,)) event = server.recv() - print event + print(event) assert event.name == 'myevent' + str(i) - assert event.args == (i,) + assert list(event.args) == [i] - server.emit('answser' + str(i * 2), (i * 2,), - xheader=dict(zmqid=event.header['zmqid'])) + reply_event = server.new_event('answser' + str(i * 2), (i * 2,)) + reply_event.identity = event.identity + server.emit_event(reply_event) event = client.recv() - print event + print(event) assert event.name == 'answser' + str(i * 2) - assert event.args == (i * 2,) + assert list(event.args) == [i * 2] def test_events_push_pull(): @@ -160,11 +165,35 @@ def test_events_push_pull(): client = zerorpc.Events(zmq.PUSH) client.connect(endpoint) - for x in xrange(10): + for x in range(10): client.emit('myevent', (x,)) - for x in xrange(10): + for x in range(10): event = server.recv() - print event + print(event) assert event.name == 'myevent' - assert event.args == (x,) + assert list(event.args) == [x] + + +def test_msgpack(): + context = zerorpc.Context() + event = zerorpc.Event(u'myevent', (u'a',), context=context) + print(event) + # note here that str is an unicode string in all Python version (thanks to + # the builtin str import). + assert isinstance(event.name, str) + for key in event.header.keys(): + assert isinstance(key, str) + assert isinstance(event.header[u'message_id'], bytes) + assert isinstance(event.header[u'v'], int) + assert isinstance(event.args[0], str) + + packed = event.pack() + event = event.unpack(packed) + print(event) + assert isinstance(event.name, str) + for key in event.header.keys(): + assert isinstance(key, str) + assert isinstance(event.header[u'message_id'], bytes) + assert isinstance(event.header[u'v'], int) + assert isinstance(event.args[0], str) diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 13caab5..14c66fd 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,39 +23,47 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + +import pytest import gevent +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_hbchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_hbchan.recv() - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_hbchan.close() - with assert_raises(zerorpc.LostRemote): - client_hbchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_hbchan.recv() + print('CLIENT LOST SERVER :)') client_hbchan.close() server.close() client.close() @@ -63,29 +71,32 @@ def test_close_server_hbchan(): def test_close_client_hbchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_hbchan.recv() - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_hbchan.close() - with assert_raises(zerorpc.LostRemote): - server_hbchan.recv() - print 'SERVER LOST CLIENT :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_hbchan.recv() + print('SERVER LOST CLIENT :)') server_hbchan.close() server.close() client.close() @@ -93,27 +104,30 @@ def test_close_client_hbchan(): def test_heartbeat_can_open_channel_server_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_hbchan.close() - with assert_raises(zerorpc.LostRemote): - client_hbchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_hbchan.recv() + print('CLIENT LOST SERVER :)') client_hbchan.close() server.close() client.close() @@ -121,61 +135,64 @@ def test_heartbeat_can_open_channel_server_close(): def test_heartbeat_can_open_channel_client_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_hbchan.close() client.close() - with assert_raises(zerorpc.LostRemote): - server_hbchan.recv() - print 'SERVER LOST CLIENT :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_hbchan.recv() + print('SERVER LOST CLIENT :)') server_hbchan.close() server.close() def test_do_some_req_rep(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 4) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 4) def client_do(): - for x in xrange(20): + for x in range(20): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) def server_do(): - for x in xrange(20): + for x in range(20): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) @@ -191,26 +208,29 @@ def server_do(): def test_do_some_req_rep_lost_server(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): - print 'running' + print('running') client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - for x in xrange(10): + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + for x in range(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_hbchan.emit('add', (x, x * x)) - with assert_raises(zerorpc.LostRemote): - event = client_hbchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_hbchan.recv() client_hbchan.close() client_task = gevent.spawn(client_do) @@ -218,8 +238,8 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - for x in xrange(10): + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + for x in range(10): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) @@ -235,23 +255,23 @@ def server_do(): def test_do_some_req_rep_lost_client(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -259,15 +279,18 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) - with assert_raises(zerorpc.LostRemote): - event = server_hbchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_hbchan.recv() server_hbchan.close() server_task = gevent.spawn(server_do) @@ -280,24 +303,33 @@ def server_do(): def test_do_some_req_rep_client_timeout(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - - with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): - client_hbchan.emit('sleep', (x,)) - event = client_hbchan.recv(timeout=3) - assert event.name == 'OK' - assert event.args == (x,) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(10): + client_hbchan.emit('sleep', (x,)) + event = client_hbchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(10): + client_hbchan.emit('sleep', (x,)) + event = client_hbchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -305,14 +337,23 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - - with assert_raises(zerorpc.LostRemote): - for x in xrange(20): - event = server_hbchan.recv() - assert event.name == 'sleep' - gevent.sleep(event.args[0]) - server_hbchan.emit('OK', event.args) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(20): + event = server_hbchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_hbchan.emit('OK', event.args) + pytest.raises(zerorpc.LostRemote, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.LostRemote): + for x in range(20): + event = server_hbchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_hbchan.emit('OK', event.args) server_hbchan.close() server_task = gevent.spawn(server_do) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 368eb17..3163a3a 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,15 +23,21 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import str +from future.utils import tobytes + +import pytest import gevent import gevent.local import random -import md5 +import hashlib +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_resolve_endpoint(): @@ -46,16 +52,16 @@ def resolve(endpoint): cnt = c.register_middleware({ 'resolve_endpoint': resolve }) - print 'registered_count:', cnt + print('registered_count:', cnt) assert cnt == 1 - print 'resolve titi:', c.hook_resolve_endpoint('titi') + print('resolve titi:', c.hook_resolve_endpoint('titi')) assert c.hook_resolve_endpoint('titi') == test_endpoint - print 'resolve toto:', c.hook_resolve_endpoint('toto') + print('resolve toto:', c.hook_resolve_endpoint('toto')) assert c.hook_resolve_endpoint('toto') == 'toto' - class Resolver(): + class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'toto': @@ -63,18 +69,18 @@ def resolve_endpoint(self, endpoint): return endpoint cnt = c.register_middleware(Resolver()) - print 'registered_count:', cnt + print('registered_count:', cnt) assert cnt == 1 - print 'resolve titi:', c.hook_resolve_endpoint('titi') + print('resolve titi:', c.hook_resolve_endpoint('titi')) assert c.hook_resolve_endpoint('titi') == test_endpoint - print 'resolve toto:', c.hook_resolve_endpoint('toto') + print('resolve toto:', c.hook_resolve_endpoint('toto')) assert c.hook_resolve_endpoint('toto') == test_endpoint c2 = zerorpc.Context() - print 'resolve titi:', c2.hook_resolve_endpoint('titi') + print('resolve titi:', c2.hook_resolve_endpoint('titi')) assert c2.hook_resolve_endpoint('titi') == 'titi' - print 'resolve toto:', c2.hook_resolve_endpoint('toto') + print('resolve toto:', c2.hook_resolve_endpoint('toto')) assert c2.hook_resolve_endpoint('toto') == 'toto' @@ -82,7 +88,7 @@ def test_resolve_endpoint_events(): test_endpoint = random_ipc_endpoint() c = zerorpc.Context() - class Resolver(): + class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'some_service': return test_endpoint @@ -90,19 +96,22 @@ def resolve_endpoint(self, endpoint): class Srv(zerorpc.Server): def hello(self): - print 'heee' + print('heee') return 'world' - srv = Srv(heartbeat=1, context=c) - with assert_raises(zmq.ZMQError): - srv.bind('some_service') + srv = Srv(heartbeat=TIME_FACTOR * 1, context=c) + if sys.version_info < (2, 7): + pytest.raises(zmq.ZMQError, srv.bind, 'some_service') + else: + with pytest.raises(zmq.ZMQError): + srv.bind('some_service') cnt = c.register_middleware(Resolver()) assert cnt == 1 srv.bind('some_service') gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=1, context=c) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1, context=c) client.connect('some_service') assert client.hello() == 'world' @@ -110,7 +119,7 @@ def hello(self): srv.close() -class Tracer: +class Tracer(object): '''Used by test_task_context_* tests''' def __init__(self, identity): self._identity = identity @@ -123,19 +132,19 @@ def trace_id(self): def load_task_context(self, event_header): self._locals.trace_id = event_header.get('trace_id', None) - print self._identity, 'load_task_context', self.trace_id + print(self._identity, 'load_task_context', self.trace_id) self._log.append(('load', self.trace_id)) def get_task_context(self): if self.trace_id is None: # just an ugly code to generate a beautiful little hash. - self._locals.trace_id = '<{0}>'.format(md5.md5( - str(random.random())[3:] + self._locals.trace_id = '<{0}>'.format(hashlib.md5( + tobytes(str(random.random())[3:]) ).hexdigest()[0:6].upper()) - print self._identity, 'get_task_context! [make a new one]', self.trace_id + print(self._identity, 'get_task_context! [make a new one]', self.trace_id) self._log.append(('new', self.trace_id)) else: - print self._identity, 'get_task_context! [reuse]', self.trace_id + print(self._identity, 'get_task_context! [reuse]', self.trace_id) self._log.append(('reuse', self.trace_id)) return { 'trace_id': self.trace_id } @@ -150,7 +159,7 @@ def test_task_context(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -197,7 +206,7 @@ def test_task_context_relay(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -208,7 +217,7 @@ def echo(self, msg): c_relay = zerorpc.Client(context=srv_relay_ctx) c_relay.connect(endpoint1) - class SrvRelay: + class SrvRelay(object): def echo(self, msg): return c_relay.echo('relay' + msg) + 'relayed' @@ -253,7 +262,7 @@ def test_task_context_relay_fork(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -264,15 +273,15 @@ def echo(self, msg): c_relay = zerorpc.Client(context=srv_relay_ctx) c_relay.connect(endpoint1) - class SrvRelay: + class SrvRelay(object): def echo(self, msg): def dothework(msg): return c_relay.echo(msg) + 'relayed' g = gevent.spawn(zerorpc.fork_task_context(dothework, srv_relay_ctx), 'relay' + msg) - print 'relaying in separate task:', g + print('relaying in separate task:', g) r = g.get() - print 'back to main task' + print('back to main task') return r srv_relay = zerorpc.Server(SrvRelay(), context=srv_relay_ctx) @@ -317,7 +326,7 @@ def test_task_context_pushpull(): trigger = gevent.event.Event() - class Puller: + class Puller(object): def echo(self, msg): trigger.set() @@ -355,7 +364,7 @@ def test_task_context_pubsub(): trigger = gevent.event.Event() - class Subscriber: + class Subscriber(object): def echo(self, msg): trigger.set() @@ -369,20 +378,18 @@ def echo(self, msg): trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). - for attempt in xrange(0, 10): + while not trigger.is_set(): c.echo('pub...') - if trigger.wait(0.2): + if trigger.wait(TIME_FACTOR * 1): break subscriber.stop() subscriber_task.join() - assert publisher_tracer._log == [ - ('new', publisher_tracer.trace_id), - ] - assert subscriber_tracer._log == [ - ('load', publisher_tracer.trace_id), - ] + print(publisher_tracer._log) + assert ('new', publisher_tracer.trace_id) in publisher_tracer._log + print(subscriber_tracer._log) + assert ('load', publisher_tracer.trace_id) in subscriber_tracer._log class InspectExceptionMiddleware(Tracer): @@ -455,7 +462,7 @@ def test_server_inspect_exception_middleware_puller(): barrier.clear() client.echo('This is a test which should call the InspectExceptionMiddleware') - barrier.wait(timeout=2) + barrier.wait(timeout=TIME_FACTOR * 2) client.close() server.close() diff --git a/tests/test_middleware_before_after_exec.py b/tests/test_middleware_before_after_exec.py index 23c2e92..5dafeb0 100644 --- a/tests/test_middleware_before_after_exec.py +++ b/tests/test_middleware_before_after_exec.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,10 +22,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import +from builtins import range + import gevent import zerorpc -from testutils import random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR class EchoModule(object): @@ -42,7 +45,7 @@ def echo(self, msg): @zerorpc.stream def echoes(self, msg): self.last_msg = 'echo: ' + msg - for i in xrange(0, 3): + for i in range(0, 3): yield self.last_msg class ServerBeforeExecMiddleware(object): @@ -91,7 +94,7 @@ def test_hook_server_before_exec_puller(): # Test without a middleware test_client.echo("test") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test" trigger.clear() @@ -100,7 +103,7 @@ def test_hook_server_before_exec_puller(): zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False test_client.echo("test with a middleware") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test with a middleware" assert test_middleware.called == True @@ -183,7 +186,7 @@ def test_hook_server_after_exec_puller(): # Test without a middleware test_client.echo("test") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test" trigger.clear() @@ -192,7 +195,7 @@ def test_hook_server_after_exec_puller(): zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False test_client.echo("test with a middleware") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test with a middleware" assert test_middleware.called == True assert test_middleware.request_event_name == 'echo' @@ -288,7 +291,7 @@ def test_hook_server_after_exec_on_error_puller(): assert test_middleware.called == False try: test_client.echo("test with a middleware") - trigger.wait(timeout=2) + trigger.wait(timeout=TIME_FACTOR * 2) except zerorpc.RemoteError: pass assert echo_module.last_msg == "Raise" diff --git a/tests/test_middleware_client.py b/tests/test_middleware_client.py index 0315100..943985e 100644 --- a/tests/test_middleware_client.py +++ b/tests/test_middleware_client.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,10 +22,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import absolute_import +from builtins import range + import gevent import zerorpc -from testutils import random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR class EchoModule(object): @@ -42,7 +45,7 @@ def echo(self, msg): @zerorpc.stream def echoes(self, msg): self.last_msg = "echo: " + msg - for i in xrange(0, 3): + for i in range(0, 3): yield self.last_msg def crash(self, msg): @@ -59,7 +62,7 @@ def echoes_crash(self, msg): def timeout(self, msg): self.last_msg = "timeout: " + msg - gevent.sleep(2) + gevent.sleep(TIME_FACTOR * 2) def test_hook_client_before_request(): @@ -175,7 +178,7 @@ def client_after_request(self, req_event, rep_event, exception): test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) - test_client = zerorpc.Client(timeout=1, context=zero_ctx) + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False @@ -211,7 +214,7 @@ def test_hook_client_after_request_remote_error(): test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) - test_client = zerorpc.Client(timeout=1, context=zero_ctx) + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False @@ -234,7 +237,7 @@ def test_hook_client_after_request_remote_error_stream(): test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) - test_client = zerorpc.Client(timeout=1, context=zero_ctx) + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False diff --git a/tests/test_pubpush.py b/tests/test_pubpush.py index 7975cee..a99f9b4 100644 --- a/tests/test_pubpush.py +++ b/tests/test_pubpush.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,11 +23,15 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + import gevent import gevent.event import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_pushpull_inheritance(): @@ -39,7 +43,7 @@ def test_pushpull_inheritance(): class Puller(zerorpc.Puller): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -50,7 +54,7 @@ def lolita(self, a, b): trigger.clear() pusher.lolita(1, 2) trigger.wait() - print 'done' + print('done') def test_pubsub_inheritance(): @@ -62,7 +66,7 @@ def test_pubsub_inheritance(): class Subscriber(zerorpc.Subscriber): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -73,10 +77,10 @@ def lolita(self, a, b): trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). - for attempt in xrange(0, 10): + for attempt in range(0, 10): publisher.lolita(1, 2) if trigger.wait(0.2): - print 'done' + print('done') return raise RuntimeError("The subscriber didn't receive any published message") @@ -87,7 +91,7 @@ def test_pushpull_composite(): class Puller(object): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -102,7 +106,7 @@ def lolita(self, a, b): trigger.clear() pusher.lolita(1, 2) trigger.wait() - print 'done' + print('done') def test_pubsub_composite(): @@ -111,7 +115,7 @@ def test_pubsub_composite(): class Subscriber(object): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -126,10 +130,10 @@ def lolita(self, a, b): trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). - for attempt in xrange(0, 10): + for attempt in range(0, 10): publisher.lolita(1, 2) if trigger.wait(0.2): - print 'done' + print('done') return raise RuntimeError("The subscriber didn't receive any published message") diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index 6b79ddb..71e1511 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,10 +23,21 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR + +try: + # Try collections.abc for 3.4+ + from collections.abc import Iterator +except ImportError: + # Fallback to collections for Python 2 + from collections import Iterator def test_rcp_streaming(): @@ -36,27 +47,27 @@ class MySrv(zerorpc.Server): @zerorpc.rep def range(self, max): - return range(max) + return list(range(max)) @zerorpc.stream def xrange(self, max): - return xrange(max) + return range(max) - srv = MySrv(heartbeat=2) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=2) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4) client.connect(endpoint) r = client.range(10) - assert r == tuple(range(10)) + assert list(r) == list(range(10)) r = client.xrange(10) - assert getattr(r, 'next', None) is not None + assert isinstance(r, Iterator) l = [] - print 'wait 4s for fun' - gevent.sleep(4) + print('wait 4s for fun') + gevent.sleep(TIME_FACTOR * 4) for x in r: l.append(x) - assert l == range(10) + assert l == list(range(10)) diff --git a/tests/test_server.py b/tests/test_server.py index 3f74679..86997a9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,12 +23,17 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + +import pytest import gevent +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_server_manual(): @@ -46,20 +51,20 @@ def add(self, a, b): srv.bind(endpoint) gevent.spawn(srv.run) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() client_channel.emit('lolita', tuple()) event = client_channel.recv() - assert event.args == (42,) + assert list(event.args) == [42] client_channel.close() client_channel = client.channel() client_channel.emit('add', (1, 2)) event = client_channel.recv() - assert event.args == (3,) + assert list(event.args) == [3] client_channel.close() srv.stop() @@ -82,10 +87,10 @@ def add(self, a, b): client = zerorpc.Client() client.connect(endpoint) - print client.lolita() + print(client.lolita()) assert client.lolita() == 42 - print client.add(1, 4) + print(client.add(1, 4)) assert client.add(1, 4) == 5 @@ -98,18 +103,21 @@ def lolita(self): return 42 def add(self, a, b): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) return a + b srv = MySrv() srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - with assert_raises(zerorpc.TimeoutExpired): - print client.add(1, 4) + if sys.version_info < (2, 7): + pytest.raises(zerorpc.TimeoutExpired, client.add, 1, 4) + else: + with pytest.raises(zerorpc.TimeoutExpired): + print(client.add(1, 4)) client.close() srv.close() @@ -126,12 +134,17 @@ def raise_something(self, a): srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - with assert_raises(zerorpc.RemoteError): - print client.raise_something(42) - assert client.raise_something(range(5)) == 4 + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print(client.raise_something(42)) + pytest.raises(zerorpc.RemoteError, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.RemoteError): + print(client.raise_something(42)) + assert client.raise_something(list(range(5))) == 4 client.close() srv.close() @@ -148,17 +161,22 @@ def raise_error(self): srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - with assert_raises(zerorpc.RemoteError): - print client.raise_error() + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print(client.raise_error()) + pytest.raises(zerorpc.RemoteError, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.RemoteError): + print(client.raise_error()) try: client.raise_error() except zerorpc.RemoteError as e: - print 'got that:', e - print 'name', e.name - print 'msg', e.msg + print('got that:', e) + print('name', e.name) + print('msg', e.msg) assert e.name == 'RuntimeError' assert e.msg == 'oops!' @@ -176,27 +194,27 @@ class MySrv(zerorpc.Server): srv.bind(endpoint) gevent.spawn(srv.run) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) rpccall = client.channel() rpccall.emit('donotexist', tuple()) event = rpccall.recv() - print event + print(event) assert event.name == 'ERR' (name, msg, tb) = event.args - print 'detailed error', name, msg, tb + print('detailed error', name, msg, tb) assert name == 'NameError' assert msg == 'donotexist' rpccall = client.channel() - rpccall.emit('donotexist', tuple(), xheader=dict(v=1)) + rpccall.emit('donotexist', tuple(), xheader={'v': 1}) event = rpccall.recv() - print event + print(event) assert event.name == 'ERR' (msg,) = event.args - print 'msg only', msg + print('msg only', msg) assert msg == "NameError('donotexist',)" client_events.close() diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py deleted file mode 100644 index c631f6d..0000000 --- a/tests/test_wrapped_events.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- coding: utf-8 -*- -# Open Source Initiative OSI - The MIT License (MIT):Licensing -# -# The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import random - -from zerorpc import zmq -import zerorpc -from testutils import teardown, random_ipc_endpoint - - -def test_sub_events(): - endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) - server_events.bind(endpoint) - server = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.XREQ) - client_events.connect(endpoint) - client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - - client_channel = client.channel() - client_channel_events = zerorpc.WrappedEvents(client_channel) - client_channel_events.emit('coucou', 42) - - event = server.recv() - print event - assert type(event.args) is tuple - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - server_channel_channel = zerorpc.ChannelMultiplexer(server_channel_events) - event = server_channel_channel.recv() - print event - assert event.name == 'coucou' - assert event.args == 42 - - server_events.close() - client_events.close() - - -def test_multiple_sub_events(): - endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) - server_events.bind(endpoint) - server = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.XREQ) - client_events.connect(endpoint) - client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - - client_channel1 = client.channel() - client_channel_events1 = zerorpc.WrappedEvents(client_channel1) - client_channel2 = client.channel() - client_channel_events2 = zerorpc.WrappedEvents(client_channel2) - client_channel_events1.emit('coucou1', 43) - client_channel_events2.emit('coucou2', 44) - client_channel_events2.emit('another', 42) - - event = server.recv() - print event - assert type(event.args) is tuple - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - event = server_channel_events.recv() - print event - assert event.name == 'coucou1' - assert event.args == 43 - - event = server.recv() - print event - assert type(event.args) is tuple - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - event = server_channel_events.recv() - print event - assert event.name == 'coucou2' - assert event.args == 44 - - event = server_channel_events.recv() - print event - assert event.name == 'another' - assert event.args == 42 - - server_events.close() - client_events.close() - - -def test_recursive_multiplexer(): - endpoint = random_ipc_endpoint() - - server_events = zerorpc.Events(zmq.XREP) - server_events.bind(endpoint) - servermux = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.XREQ) - client_events.connect(endpoint) - clientmux = zerorpc.ChannelMultiplexer(client_events, - ignore_broadcast=True) - - def ping_pong(climux, srvmux): - cli_chan = climux.channel() - someid = random.randint(0, 1000000) - print 'ping...' - cli_chan.emit('ping', someid) - print 'srv_chan got:' - event = srvmux.recv() - srv_chan = srvmux.channel(event) - print event - assert event.name == 'ping' - assert event.args == someid - print 'pong...' - srv_chan.emit('pong', someid) - print 'cli_chan got:' - event = cli_chan.recv() - print event - assert event.name == 'pong' - assert event.args == someid - srv_chan.close() - cli_chan.close() - - def create_sub_multiplexer(events, from_event=None, - ignore_broadcast=False): - channel = events.channel(from_event) - sub_events = zerorpc.WrappedEvents(channel) - sub_multiplexer = zerorpc.ChannelMultiplexer(sub_events, - ignore_broadcast=ignore_broadcast) - return sub_multiplexer - - def open_sub_multiplexer(climux, srvmux): - someid = random.randint(0, 1000000) - print 'open...' - clisubmux = create_sub_multiplexer(climux, ignore_broadcast=True) - clisubmux.emit('open that', someid) - print 'srvsubmux got:' - event = srvmux.recv() - assert event.name == 'w' - srvsubmux = create_sub_multiplexer(srvmux, event) - event = srvsubmux.recv() - print event - return (clisubmux, srvsubmux) - - ping_pong(clientmux, servermux) - - (clientmux_lv2, servermux_lv2) = open_sub_multiplexer(clientmux, servermux) - ping_pong(clientmux_lv2, servermux_lv2) - - (clientmux_lv3, servermux_lv3) = open_sub_multiplexer(clientmux_lv2, - servermux_lv2) - ping_pong(clientmux_lv3, servermux_lv3) - - (clientmux_lv4, servermux_lv4) = open_sub_multiplexer(clientmux_lv3, - servermux_lv3) - ping_pong(clientmux_lv4, servermux_lv4) - - ping_pong(clientmux_lv4, servermux_lv4) - ping_pong(clientmux_lv3, servermux_lv3) - ping_pong(clientmux_lv2, servermux_lv2) - ping_pong(clientmux, servermux) - ping_pong(clientmux, servermux) - ping_pong(clientmux_lv2, servermux_lv2) - ping_pong(clientmux_lv4, servermux_lv4) - ping_pong(clientmux_lv3, servermux_lv3) - - (clientmux_lv5, servermux_lv5) = open_sub_multiplexer(clientmux_lv4, - servermux_lv4) - ping_pong(clientmux_lv5, servermux_lv5) diff --git a/tests/test_zmq.py b/tests/test_zmq.py index 6e9dad1..1e7b4dd 100644 --- a/tests/test_zmq.py +++ b/tests/test_zmq.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,22 +23,26 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import import gevent from zerorpc import zmq +from .testutils import teardown, random_ipc_endpoint def test1(): + endpoint = random_ipc_endpoint() def server(): c = zmq.Context() s = c.socket(zmq.REP) - s.bind('tcp://0.0.0.0:9999') + s.bind(endpoint) while True: - print 'srv recving...' + print(b'srv recving...') r = s.recv() - print 'srv', r - print 'srv sending...' - s.send('world') + print('srv', r) + print('srv sending...') + s.send(b'world') s.close() c.term() @@ -46,13 +50,13 @@ def server(): def client(): c = zmq.Context() s = c.socket(zmq.REQ) - s.connect('tcp://localhost:9999') + s.connect(endpoint) - print 'cli sending...' - s.send('hello') - print 'cli recving...' + print('cli sending...') + s.send(b'hello') + print('cli recving...') r = s.recv() - print 'cli', r + print('cli', r) s.close() c.term() diff --git a/tests/testutils.py b/tests/testutils.py index 175967c..85b6a96 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,8 +22,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import print_function +from builtins import str + import functools -import nose.exc +import pytest import random import os @@ -38,7 +41,7 @@ def random_ipc_endpoint(): def teardown(): global _tmpfiles for tmpfile in _tmpfiles: - print 'unlink', tmpfile + print('unlink', tmpfile) try: os.unlink(tmpfile) except Exception: @@ -49,6 +52,12 @@ def skip(reason): def _skip(test): @functools.wraps(test) def wrap(): - raise nose.exc.SkipTest(reason) + raise pytest.SkipTest(reason) return wrap return _skip + +try: + TIME_FACTOR = float(os.environ.get('ZPC_TEST_TIME_FACTOR')) +except TypeError: + TIME_FACTOR = 0.2 +print('ZPC_TEST_TIME_FACTOR:', TIME_FACTOR) diff --git a/tests/zmqbug.py b/tests/zmqbug.py index 2d385f0..1d102a2 100644 --- a/tests/zmqbug.py +++ b/tests/zmqbug.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -25,6 +25,8 @@ # Based on https://github.com/traviscline/gevent-zeromq/blob/master/gevent_zeromq/core.py +from __future__ import print_function + import zmq import gevent.event @@ -79,7 +81,7 @@ def send(self, data, flags=0, copy=True, track=False): while True: try: return super(ZMQSocket, self).send(data, flags, copy, track) - except zmq.ZMQError, e: + except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: raise self._writable.clear() @@ -92,14 +94,14 @@ def recv(self, flags=0, copy=True, track=False): while True: try: return super(ZMQSocket, self).recv(flags, copy, track) - except zmq.ZMQError, e: + except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: raise self._readable.clear() while not self._readable.wait(timeout=10): events = self.getsockopt(zmq.EVENTS) if bool(events & zmq.POLLIN): - print "here we go, nobody told me about new messages!" + print("here we go, nobody told me about new messages!") global STOP_EVERYTHING STOP_EVERYTHING = True raise gevent.GreenletExit() @@ -111,7 +113,7 @@ def server(): socket = ZMQSocket(zmq_context, zmq.REP) socket.bind('ipc://zmqbug') - class Cnt: + class Cnt(object): responded = 0 cnt = Cnt() @@ -125,15 +127,15 @@ def responder(): gevent.spawn(responder) while not STOP_EVERYTHING: - print "cnt.responded=", cnt.responded + print("cnt.responded=", cnt.responded) gevent.sleep(0.5) def client(): - socket = ZMQSocket(zmq_context, zmq.XREQ) + socket = ZMQSocket(zmq_context, zmq.DEALER) socket.connect('ipc://zmqbug') - class Cnt: + class Cnt(object): recv = 0 send = 0 @@ -156,7 +158,7 @@ def sendmsg(): gevent.spawn(sendmsg) while not STOP_EVERYTHING: - print "cnt.recv=", cnt.recv, "cnt.send=", cnt.send + print("cnt.recv=", cnt.recv, "cnt.send=", cnt.send) gevent.sleep(0.5) gevent.spawn(server) diff --git a/tox.ini b/tox.ini index a9a789f..96bace8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,16 @@ [tox] -envlist = py26,py27,py31,py32 +envlist = py26,py27,py34,py35 [testenv] -deps=nose -commands=nosetests +deps = + flake8 + pytest +commands = + flake8 zerorpc bin + pytest -v +passenv = ZPC_TEST_TIME_FACTOR + +[flake8] +ignore = E501,E128 +filename = *.py,zerorpc +exclude = tests,.git,dist,doc,*.egg-info,__pycache__,setup.py diff --git a/zerorpc/__init__.py b/zerorpc/__init__.py index 4e1b040..23e6894 100644 --- a/zerorpc/__init__.py +++ b/zerorpc/__init__.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,6 +22,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# Tell flake8 to ignore this file (otherwise it will complain about import *) +# flake8: noqa from .version import * from .exceptions import * from .context import * diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 199900b..ad21c27 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,65 +22,64 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import sys - import gevent.pool import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock +import logging from .exceptions import TimeoutExpired +from .channel_base import ChannelBase + + +logger = logging.getLogger(__name__) -class ChannelMultiplexer(object): +class ChannelMultiplexer(ChannelBase): def __init__(self, events, ignore_broadcast=False): self._events = events self._active_channels = {} self._channel_dispatcher_task = None self._broadcast_queue = None - if events.recv_is_available and not ignore_broadcast: + if events.recv_is_supported and not ignore_broadcast: self._broadcast_queue = gevent.queue.Queue(maxsize=1) self._channel_dispatcher_task = gevent.spawn( self._channel_dispatcher) @property - def recv_is_available(self): - return self._events.recv_is_available + def recv_is_supported(self): + return self._events.recv_is_supported - def __del__(self): - self.close() + @property + def emit_is_supported(self): + return self._events.emit_is_supported def close(self): if self._channel_dispatcher_task: self._channel_dispatcher_task.kill() - def create_event(self, name, args, xheader={}): - return self._events.create_event(name, args, xheader) - - def emit_event(self, event, identity=None): - return self._events.emit_event(event, identity) + def new_event(self, name, args, xheader=None): + return self._events.new_event(name, args, xheader) - def emit(self, name, args, xheader={}): - return self._events.emit(name, args, xheader) + def emit_event(self, event, timeout=None): + return self._events.emit_event(event, timeout) - def recv(self): + def recv(self, timeout=None): if self._broadcast_queue is not None: - event = self._broadcast_queue.get() + event = self._broadcast_queue.get(timeout=timeout) else: - event = self._events.recv() + event = self._events.recv(timeout=timeout) return event def _channel_dispatcher(self): while True: try: event = self._events.recv() - except Exception as e: - print >> sys.stderr, \ - 'zerorpc.ChannelMultiplexer,', \ - 'ignoring error on recv: {0}'.format(e) + except Exception: + logger.exception('zerorpc.ChannelMultiplexer ignoring error on recv') continue - channel_id = event.header.get('response_to', None) + channel_id = event.header.get(u'response_to', None) queue = None if channel_id is not None: @@ -91,10 +90,9 @@ def _channel_dispatcher(self): queue = self._broadcast_queue if queue is None: - print >> sys.stderr, \ - 'zerorpc.ChannelMultiplexer,', \ - 'unable to route event:', \ - event.__str__(ignore_args=True) + logger.warning('zerorpc.ChannelMultiplexer,' + ' unable to route event: {0}'.format( + event.__str__(ignore_args=True))) else: queue.put(event) @@ -113,7 +111,7 @@ def context(self): return self._events.context -class Channel(object): +class Channel(ChannelBase): def __init__(self, multiplexer, from_event=None): self._multiplexer = multiplexer @@ -121,38 +119,39 @@ def __init__(self, multiplexer, from_event=None): self._zmqid = None self._queue = gevent.queue.Queue(maxsize=1) if from_event is not None: - self._channel_id = from_event.header['message_id'] - self._zmqid = from_event.header.get('zmqid', None) + self._channel_id = from_event.header[u'message_id'] + self._zmqid = from_event.identity self._multiplexer._active_channels[self._channel_id] = self + logger.debug('<-- new channel %s', self._channel_id) self._queue.put(from_event) @property - def recv_is_available(self): - return self._multiplexer.recv_is_available + def recv_is_supported(self): + return self._multiplexer.recv_is_supported - def __del__(self): - self.close() + @property + def emit_is_supported(self): + return self._multiplexer.emit_is_supported def close(self): if self._channel_id is not None: del self._multiplexer._active_channels[self._channel_id] + logger.debug('-x- closed channel %s', self._channel_id) self._channel_id = None - def create_event(self, name, args, xheader={}): - event = self._multiplexer.create_event(name, args, xheader) + def new_event(self, name, args, xheader=None): + event = self._multiplexer.new_event(name, args, xheader) if self._channel_id is None: - self._channel_id = event.header['message_id'] + self._channel_id = event.header[u'message_id'] self._multiplexer._active_channels[self._channel_id] = self + logger.debug('--> new channel %s', self._channel_id) else: - event.header['response_to'] = self._channel_id + event.header[u'response_to'] = self._channel_id + event.identity = self._zmqid return event - def emit(self, name, args, xheader={}): - event = self.create_event(name, args, xheader) - self._multiplexer.emit_event(event, self._zmqid) - - def emit_event(self, event): - self._multiplexer.emit_event(event, self._zmqid) + def emit_event(self, event, timeout=None): + self._multiplexer.emit_event(event, timeout) def recv(self, timeout=None): try: @@ -166,7 +165,7 @@ def context(self): return self._multiplexer.context -class BufferedChannel(object): +class BufferedChannel(ChannelBase): def __init__(self, channel, inqueue_size=100): self._channel = channel @@ -175,14 +174,17 @@ def __init__(self, channel, inqueue_size=100): self._input_queue_reserved = 1 self._remote_can_recv = gevent.event.Event() self._input_queue = gevent.queue.Queue() - self._lost_remote = False self._verbose = False self._on_close_if = None self._recv_task = gevent.spawn(self._recver) @property - def recv_is_available(self): - return self._channel.recv_is_available + def recv_is_supported(self): + return self._channel.recv_is_supported + + @property + def emit_is_supported(self): + return self._channel.emit_is_supported @property def on_close_if(self): @@ -192,9 +194,6 @@ def on_close_if(self): def on_close_if(self, cb): self._on_close_if = cb - def __del__(self): - self.close() - def close(self): if self._recv_task is not None: self._recv_task.kill() @@ -206,18 +205,16 @@ def close(self): def _recver(self): while True: event = self._channel.recv() - if event.name == '_zpc_more': + if event.name == u'_zpc_more': try: self._remote_queue_open_slots += int(event.args[0]) - except Exception as e: - print >> sys.stderr, \ - 'gevent_zerorpc.BufferedChannel._recver,', \ - 'exception:', e + except Exception: + logger.exception('gevent_zerorpc.BufferedChannel._recver') if self._remote_queue_open_slots > 0: self._remote_can_recv.set() elif self._input_queue.qsize() == self._input_queue_size: raise RuntimeError( - 'BufferedChannel, queue overflow on event:', event) + 'BufferedChannel, queue overflow on event:', event) else: self._input_queue.put(event) if self._on_close_if is not None and self._on_close_if(event): @@ -225,13 +222,11 @@ def _recver(self): self.close() return - def create_event(self, name, args, xheader={}): - return self._channel.create_event(name, args, xheader) + def new_event(self, name, args, xheader=None): + return self._channel.new_event(name, args, xheader) - def emit_event(self, event, block=True, timeout=None): + def emit_event(self, event, timeout=None): if self._remote_queue_open_slots == 0: - if not block: - return False self._remote_can_recv.clear() self._remote_can_recv.wait(timeout=timeout) self._remote_queue_open_slots -= 1 @@ -240,20 +235,18 @@ def emit_event(self, event, block=True, timeout=None): except: self._remote_queue_open_slots += 1 raise - return True - - def emit(self, name, args, xheader={}, block=True, timeout=None): - event = self.create_event(name, args, xheader) - return self.emit_event(event, block, timeout) def _request_data(self): open_slots = self._input_queue_size - self._input_queue_reserved self._input_queue_reserved += open_slots - self._channel.emit('_zpc_more', (open_slots,)) + self._channel.emit(u'_zpc_more', (open_slots,)) def recv(self, timeout=None): - if self._verbose: - if self._input_queue_reserved < self._input_queue_size / 2: + # self._channel can be set to None by an 'on_close_if' callback if it + # sees a suitable message from the remote end... + # + if self._verbose and self._channel: + if self._input_queue_reserved < self._input_queue_size // 2: self._request_data() else: self._verbose = True diff --git a/zerorpc/channel_base.py b/zerorpc/channel_base.py new file mode 100644 index 0000000..a391b08 --- /dev/null +++ b/zerorpc/channel_base.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Open Source Initiative OSI - The MIT License (MIT):Licensing +# +# The MIT License (MIT) +# Copyright (c) 2014 François-Xavier Bourlet (bombela@gmail.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +class ChannelBase(object): + + @property + def context(self): + raise NotImplementedError() + + @property + def recv_is_supported(self): + raise NotImplementedError() + + @property + def emit_is_supported(self): + raise NotImplementedError() + + def close(self): + raise NotImplementedError() + + def new_event(self, name, args, xheader=None): + raise NotImplementedError() + + def emit_event(self, event, timeout=None): + raise NotImplementedError() + + def emit(self, name, args, xheader=None, timeout=None): + event = self.new_event(name, args, xheader) + return self.emit_event(event, timeout) + + def recv(self, timeout=None): + raise NotImplementedError() diff --git a/zerorpc/cli.py b/zerorpc/cli.py index d32ed2f..3fd6350 100644 --- a/zerorpc/cli.py +++ b/zerorpc/cli.py @@ -3,7 +3,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -24,19 +24,30 @@ # SOFTWARE. +from __future__ import print_function +from builtins import map + import argparse import json import sys import inspect import os +import logging from pprint import pprint import zerorpc +try: + # Try collections.abc for 3.4+ + from collections.abc import Iterator +except ImportError: + # Fallback to collections for Python 2 + from collections import Iterator + parser = argparse.ArgumentParser( - description='Make a zerorpc call to a remote service.' - ) + description='Make a zerorpc call to a remote service.' +) client_or_server = parser.add_mutually_exclusive_group() client_or_server.add_argument('--client', action='store_true', default=True, @@ -56,6 +67,8 @@ parser.add_argument('--heartbeat', default=5, metavar='seconds', type=int, help='heartbeat frequency. You should always use \ the same frequency as the server. (default: 5s)') +parser.add_argument('--pool-size', default=None, metavar='count', type=int, + help='size of worker pool. --server only.') parser.add_argument('-j', '--json', default=False, action='store_true', help='arguments are in JSON format and will be be parsed \ before being sent to the remote') @@ -68,6 +81,9 @@ parser.add_argument('--active-hb', default=False, action='store_true', help='enable active heartbeat. The default is to \ wait for the server to send the first heartbeat') +parser.add_argument('-d', '--debug', default=False, action='store_true', + help='Print zerorpc debug msgs, \ + like outgoing and incomming messages.') parser.add_argument('address', nargs='?', help='address to connect to. Skip \ this if you specified --connect or --bind at least once') parser.add_argument('command', nargs='?', @@ -82,7 +98,7 @@ def setup_links(args, socket): if args.bind: for endpoint in args.bind: - print 'binding to "{0}"'.format(endpoint) + print('binding to "{0}"'.format(endpoint), file=sys.stderr) socket.bind(endpoint) addresses = [] if args.address: @@ -90,7 +106,7 @@ def setup_links(args, socket): if args.connect: addresses.extend(args.connect) for endpoint in addresses: - print 'connecting to "{0}"'.format(endpoint) + print('connecting to "{0}"'.format(endpoint), file=sys.stderr) socket.connect(endpoint) @@ -108,9 +124,11 @@ def run_server(args): if callable(server_obj): server_obj = server_obj() - server = zerorpc.Server(server_obj, heartbeat=args.heartbeat) + server = zerorpc.Server(server_obj, heartbeat=args.heartbeat, pool_size=args.pool_size) + if args.debug: + server.debug = True setup_links(args, server) - print 'serving "{0}"'.format(server_obj_path) + print('serving "{0}"'.format(server_obj_path), file=sys.stderr) return server.run() @@ -139,13 +157,14 @@ def remote_detailled_methods(): remote_detailled_methods())) r = [(name + (inspect.formatargspec(*argspec) - if argspec else '(...)'), doc) - for name, argspec, doc in remote_detailled_methods()] - longest_name_len = max(len(name) for name, doc in r) + if argspec else '(...)'), doc) + for name, argspec, doc in remote_detailled_methods()] + longest_name_len = max(len(name) for name, doc in r) if r else 0 return (longest_name_len, r) + # handle the 'python formatted' _zerorpc_inspect, that return the output of -# "getargspec" from the python lib "inspect". +# "getargspec" from the python lib "inspect". A monstruosity from protocol v2. def zerorpc_inspect_python_argspecs(remote_methods, filter_method, long_doc, include_argspec): def format_method(name, argspec, doc): if include_argspec: @@ -157,10 +176,14 @@ def format_method(name, argspec, doc): doc = doc.splitlines()[0] return (name, doc) r = [format_method(*methods_info) for methods_info in remote_methods if - filter_method is None or methods_info[0] == filter_method] - longest_name_len = max(len(name) for name, doc in r) + filter_method is None or methods_info[0] == filter_method] + if not r: + return None + longest_name_len = max(len(name) for name, doc in r) if r else 0 return (longest_name_len, r) + +# Handles generically formatted arguments (not tied to any specific programming language). def zerorpc_inspect_generic(remote_methods, filter_method, long_doc, include_argspec): def format_method(name, args, doc): if include_argspec: @@ -170,63 +193,87 @@ def format_arg(arg): return arg['name'] return '{0}={1}'.format(arg['name'], def_val) - name += '({0})'.format(', '.join(map(format_arg, args))) + if args: + name += '({0})'.format(', '.join(map(format_arg, args))) + else: + name += '(??)' + if not doc: doc = '' elif not long_doc: doc = doc.splitlines()[0] return (name, doc) - methods = [format_method(name, details['args'], details['doc']) for name, details in remote_methods.items() + methods = [format_method(name, details['args'], details['doc']) + for name, details in remote_methods.items() if filter_method is None or name == filter_method] - longest_name_len = max(len(name) for name, doc in methods) + longest_name_len = (max(len(name) for name, doc in methods) + if methods else 0) return (longest_name_len, methods) + def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): try: - remote_methods = client._zerorpc_inspect()['methods'] + inspect_result = client._zerorpc_inspect() + remote_methods = inspect_result['methods'] legacy = False except (zerorpc.RemoteError, NameError): legacy = True if legacy: - return zerorpc_inspect_legacy(client, method, - long_doc, include_argspec) + try: + service_name = client._zerorpc_name() + except (zerorpc.RemoteError): + service_name = 'N/A' - if not isinstance(remote_methods, dict): - return zerorpc_inspect_python_argspecs(remote_methods, method, long_doc, - include_argspec) + (longest_name_len, detailled_methods) = zerorpc_inspect_legacy(client, + method, long_doc, include_argspec) + else: + service_name = inspect_result.get('name', 'N/A') + if not isinstance(remote_methods, dict): + (longest_name_len, + detailled_methods) = zerorpc_inspect_python_argspecs( + remote_methods, method, long_doc, include_argspec) + + (longest_name_len, detailled_methods) = zerorpc_inspect_generic( + remote_methods, method, long_doc, include_argspec) + + return longest_name_len, detailled_methods, service_name - return zerorpc_inspect_generic(remote_methods, method, long_doc, - include_argspec) def run_client(args): client = zerorpc.Client(timeout=args.timeout, heartbeat=args.heartbeat, passive_heartbeat=not args.active_hb) + if args.debug: + client.debug = True setup_links(args, client) if not args.command: - (longest_name_len, detailled_methods) = zerorpc_inspect(client, + (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, long_doc=False, include_argspec=args.inspect) + print('[{0}]'.format(service)) if args.inspect: for (name, doc) in detailled_methods: - print name + print(name) else: for (name, doc) in detailled_methods: - print '{0} {1}'.format(name.ljust(longest_name_len), doc) + print('{0} {1}'.format(name.ljust(longest_name_len), doc)) return if args.inspect: - (longest_name_len, detailled_methods) = zerorpc_inspect(client, + (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, method=args.command) - (name, doc) = detailled_methods[0] - print '\n{0}\n\n{1}\n'.format(name, doc) + if detailled_methods: + (name, doc) = detailled_methods[0] + print('[{0}]\n{1}\n\n{2}\n'.format(service, name, doc)) + else: + print('[{0}]\nNo documentation for "{1}".'.format(service, args.command)) return if args.json: call_args = [json.loads(x) for x in args.params] else: call_args = args.params results = client(args.command, *call_args) - if getattr(results, 'next', None) is None: + if not isinstance(results, Iterator): if args.print_json: json.dump(results, sys.stdout) else: @@ -249,8 +296,12 @@ def run_client(args): def main(): + logging.basicConfig() args = parser.parse_args() + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + if args.bind or args.connect: if args.command: args.params.insert(0, args.command) @@ -269,4 +320,3 @@ def main(): return -1 return run_server(args) - diff --git a/zerorpc/context.py b/zerorpc/context.py index 7c53a36..debce26 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,16 +23,20 @@ # SOFTWARE. +from __future__ import absolute_import +from future.utils import tobytes + import uuid import random -import gevent_zmq as zmq +from . import gevent_zmq as zmq class Context(zmq.Context): _instance = None def __init__(self): + super(zmq.Context, self).__init__() self._middlewares = [] self._hooks = { 'resolve_endpoint': [], @@ -43,10 +47,55 @@ def __init__(self): 'server_inspect_exception': [], 'client_handle_remote_error': [], 'client_before_request': [], - 'client_after_request': [] + 'client_after_request': [], + 'client_patterns_list': [], } self._reset_msgid() + # NOTE: pyzmq 13.0.0 messed up with setattr (they turned it into a + # non-op) and you can't assign attributes normally anymore, hence the + # tricks with self.__dict__ here + + @property + def _middlewares(self): + return self.__dict__['_middlewares'] + + @_middlewares.setter + def _middlewares(self, value): + self.__dict__['_middlewares'] = value + + @property + def _hooks(self): + return self.__dict__['_hooks'] + + @_hooks.setter + def _hooks(self, value): + self.__dict__['_hooks'] = value + + @property + def _msg_id_base(self): + return self.__dict__['_msg_id_base'] + + @_msg_id_base.setter + def _msg_id_base(self, value): + self.__dict__['_msg_id_base'] = value + + @property + def _msg_id_counter(self): + return self.__dict__['_msg_id_counter'] + + @_msg_id_counter.setter + def _msg_id_counter(self, value): + self.__dict__['_msg_id_counter'] = value + + @property + def _msg_id_counter_stop(self): + return self.__dict__['_msg_id_counter_stop'] + + @_msg_id_counter_stop.setter + def _msg_id_counter_stop(self, value): + self.__dict__['_msg_id_counter_stop'] = value + @staticmethod def get_instance(): if Context._instance is None: @@ -54,21 +103,21 @@ def get_instance(): return Context._instance def _reset_msgid(self): - self._msg_id_base = str(uuid.uuid4())[8:] - self._msg_id_counter = random.randrange(0, 2**32) - self._msg_id_counter_stop = random.randrange(self._msg_id_counter, 2**32) + self._msg_id_base = tobytes(uuid.uuid4().hex)[8:] + self._msg_id_counter = random.randrange(0, 2 ** 32) + self._msg_id_counter_stop = random.randrange(self._msg_id_counter, 2 ** 32) def new_msgid(self): if self._msg_id_counter >= self._msg_id_counter_stop: self._reset_msgid() else: self._msg_id_counter = (self._msg_id_counter + 1) - return '{0:08x}{1}'.format(self._msg_id_counter, self._msg_id_base) + return tobytes('{0:08x}'.format(self._msg_id_counter)) + self._msg_id_base def register_middleware(self, middleware_instance): registered_count = 0 self._middlewares.append(middleware_instance) - for hook in self._hooks.keys(): + for hook in self._hooks: functor = getattr(middleware_instance, hook, None) if functor is None: try: @@ -80,9 +129,9 @@ def register_middleware(self, middleware_instance): registered_count += 1 return registered_count - + # # client/server - + # def hook_resolve_endpoint(self, endpoint): for functor in self._hooks['resolve_endpoint']: endpoint = functor(endpoint) @@ -98,9 +147,9 @@ def hook_get_task_context(self): event_header.update(functor()) return event_header - + # # Server-side hooks - + # def hook_server_before_exec(self, request_event): """Called when a method is about to be executed on the server.""" @@ -131,9 +180,9 @@ def hook_server_inspect_exception(self, request_event, reply_event, exc_infos): for functor in self._hooks['server_inspect_exception']: functor(request_event, reply_event, task_context, exc_infos) - + # # Client-side hooks - + # def hook_client_handle_remote_error(self, event): exception = None for functor in self._hooks['client_handle_remote_error']: @@ -172,3 +221,8 @@ def hook_client_after_request(self, request_event, reply_event, exception=None): """ for functor in self._hooks['client_after_request']: functor(request_event, reply_event, exception) + + def hook_client_patterns_list(self, patterns): + for functor in self._hooks['client_patterns_list']: + patterns = functor(patterns) + return patterns diff --git a/zerorpc/core.py b/zerorpc/core.py index 5e0b0eb..ec2e008 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,22 +23,31 @@ # SOFTWARE. +from __future__ import absolute_import +from builtins import str +from builtins import zip +from future.utils import iteritems + import sys import traceback import gevent.pool import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock -import gevent_zmq as zmq +from . import gevent_zmq as zmq from .exceptions import TimeoutExpired, RemoteError, LostRemote from .channel import ChannelMultiplexer, BufferedChannel from .socket import SocketBase from .heartbeat import HeartBeatOnChannel from .context import Context from .decorators import DecoratorBase, rep -import patterns +from . import patterns +from logging import getLogger + +logger = getLogger(__name__) + class ServerBase(object): @@ -58,26 +67,26 @@ def __init__(self, channel, methods=None, name=None, context=None, self._inject_builtins() self._heartbeat_freq = heartbeat - for (k, functor) in self._methods.items(): + for (k, functor) in iteritems(self._methods): if not isinstance(functor, DecoratorBase): self._methods[k] = rep(functor) @staticmethod def _filter_methods(cls, self, methods): - if hasattr(methods, '__getitem__'): + if isinstance(methods, dict): return methods - server_methods = set(getattr(self, k) for k in dir(cls) if not - k.startswith('_')) + server_methods = set(k for k in dir(cls) if not k.startswith('_')) return dict((k, getattr(methods, k)) - for k in dir(methods) - if callable(getattr(methods, k)) - and not k.startswith('_') - and getattr(methods, k) not in server_methods - ) + for k in dir(methods) + if callable(getattr(methods, k)) and + not k.startswith('_') and k not in server_methods + ) @staticmethod def _extract_name(methods): - return getattr(type(methods), '__name__', None) or repr(methods) + return getattr(methods, '__name__', None) \ + or getattr(type(methods), '__name__', None) \ + or repr(methods) def close(self): self.stop() @@ -93,11 +102,11 @@ def _format_args_spec(self, args_spec, r=None): return r def _zerorpc_inspect(self): - methods = dict((m, f) for m, f in self._methods.items() + methods = dict((m, f) for m, f in iteritems(self._methods) if not m.startswith('_')) detailled_methods = dict((m, dict(args=self._format_args_spec(f._zerorpc_args()), - doc=f._zerorpc_doc())) for (m, f) in methods.items()) + doc=f._zerorpc_doc())) for (m, f) in iteritems(methods)) return {'name': self._name, 'methods': detailled_methods} @@ -107,7 +116,7 @@ def _inject_builtins(self): self._methods['_zerorpc_name'] = lambda: self._name self._methods['_zerorpc_ping'] = lambda: ['pong', self._name] self._methods['_zerorpc_help'] = lambda m: \ - self._methods[m]._zerorpc_doc() + self._methods[m]._zerorpc_doc() self._methods['_zerorpc_args'] = \ lambda m: self._methods[m]._zerorpc_args() self._methods['_zerorpc_inspect'] = self._zerorpc_inspect @@ -118,7 +127,7 @@ def __call__(self, method, *args): return self._methods[method](*args) def _print_traceback(self, protocol_v1, exc_infos): - traceback.print_exception(*exc_infos, file=sys.stderr) + logger.exception('') exc_type, exc_value, exc_traceback = exc_infos if protocol_v1: @@ -129,7 +138,7 @@ def _print_traceback(self, protocol_v1, exc_infos): return (name, human_msg, human_traceback) def _async_task(self, initial_event): - protocol_v1 = initial_event.header.get('v', 1) < 2 + protocol_v1 = initial_event.header.get(u'v', 1) < 2 channel = self._multiplexer.channel(initial_event) hbchan = HeartBeatOnChannel(channel, freq=self._heartbeat_freq, passive=protocol_v1) @@ -148,7 +157,7 @@ def _async_task(self, initial_event): except Exception: exc_infos = list(sys.exc_info()) human_exc_infos = self._print_traceback(protocol_v1, exc_infos) - reply_event = bufchan.create_event('ERR', human_exc_infos, + reply_event = bufchan.new_event(u'ERR', human_exc_infos, self._context.hook_get_task_context()) self._context.hook_server_inspect_exception(event, reply_event, exc_infos) bufchan.emit_event(reply_event) @@ -192,7 +201,7 @@ def close(self): def _handle_remote_error(self, event): exception = self._context.hook_client_handle_remote_error(event) if not exception: - if event.header.get('v', 1) >= 2: + if event.header.get(u'v', 1) >= 2: (name, msg, traceback) = event.args exception = RemoteError(name, msg, traceback) else: @@ -202,29 +211,50 @@ def _handle_remote_error(self, event): return exception def _select_pattern(self, event): - for pattern in patterns.patterns_list: + for pattern in self._context.hook_client_patterns_list( + patterns.patterns_list): if pattern.accept_answer(event): return pattern - msg = 'Unable to find a pattern for: {0}'.format(event) - raise RuntimeError(msg) + return None def _process_response(self, request_event, bufchan, timeout): - try: - reply_event = bufchan.recv(timeout) - pattern = self._select_pattern(reply_event) - return pattern.process_answer(self._context, bufchan, request_event, - reply_event, self._handle_remote_error) - except TimeoutExpired: + def raise_error(ex): bufchan.close() - ex = TimeoutExpired(timeout, - 'calling remote method {0}'.format(request_event.name)) self._context.hook_client_after_request(request_event, None, ex) raise ex - except: - bufchan.close() - raise + + try: + reply_event = bufchan.recv(timeout=timeout) + except TimeoutExpired: + raise_error(TimeoutExpired(timeout, + 'calling remote method {0}'.format(request_event.name))) + + pattern = self._select_pattern(reply_event) + if pattern is None: + raise_error(RuntimeError( + 'Unable to find a pattern for: {0}'.format(request_event))) + + return pattern.process_answer(self._context, bufchan, request_event, + reply_event, self._handle_remote_error) def __call__(self, method, *args, **kargs): + # here `method` is either a string of bytes or an unicode string in + # Python2 and Python3. Python2: str aka a byte string containing ASCII + # (unless the user explicitly provide an unicode string). Python3: str + # aka an unicode string (unless the user explicitly provide a byte + # string). + # zerorpc protocol requires an utf-8 encoded string at the msgpack + # level. msgpack will encode any unicode string object to UTF-8 and tag + # it `string`, while a bytes string will be tagged `bin`. + # + # So when we get a bytes string, we assume it to be an UTF-8 string + # (ASCII is contained in UTF-8) that we decode to an unicode string. + # Right after, msgpack-python will re-encode it as UTF-8. Yes this is + # terribly inefficient with Python2 because most of the time `method` + # will already be an UTF-8 encoded bytes string. + if isinstance(method, bytes): + method = method.decode('utf-8') + timeout = kargs.get('timeout', self._timeout) channel = self._multiplexer.channel() hbchan = HeartBeatOnChannel(channel, freq=self._heartbeat_freq, @@ -232,25 +262,20 @@ def __call__(self, method, *args, **kargs): bufchan = BufferedChannel(hbchan, inqueue_size=kargs.get('slots', 100)) xheader = self._context.hook_get_task_context() - request_event = bufchan.create_event(method, args, xheader) + request_event = bufchan.new_event(method, args, xheader) self._context.hook_client_before_request(request_event) bufchan.emit_event(request_event) - try: - if kargs.get('async', False) is False: - return self._process_response(request_event, bufchan, timeout) - - async_result = gevent.event.AsyncResult() - gevent.spawn(self._process_response, request_event, bufchan, - timeout).link(async_result) - return async_result - except: - # XXX: This is going to be closed twice if async is false and - # _process_response raises an exception. I wonder if the above - # async branch can raise an exception too, if no we can just remove - # this code. - bufchan.close() - raise + # In python 3.7, "async" is a reserved keyword, clients should now use + # "async_": support both for the time being + if (kargs.get('async', False) is False and + kargs.get('async_', False) is False): + return self._process_response(request_event, bufchan, timeout) + + async_result = gevent.event.AsyncResult() + gevent.spawn(self._process_response, request_event, bufchan, + timeout).link(async_result) + return async_result def __getattr__(self, method): return lambda *args, **kargs: self(method, *args, **kargs) @@ -260,7 +285,7 @@ class Server(SocketBase, ServerBase): def __init__(self, methods=None, name=None, context=None, pool_size=None, heartbeat=5): - SocketBase.__init__(self, zmq.XREP, context) + SocketBase.__init__(self, zmq.ROUTER, context) if methods is None: methods = self @@ -278,7 +303,7 @@ class Client(SocketBase, ClientBase): def __init__(self, connect_to=None, context=None, timeout=30, heartbeat=5, passive_heartbeat=False): - SocketBase.__init__(self, zmq.XREQ, context=context) + SocketBase.__init__(self, zmq.DEALER, context=context) ClientBase.__init__(self, self._events, context, timeout, heartbeat, passive_heartbeat) if connect_to: @@ -337,7 +362,7 @@ def _receiver(self): except Exception: exc_infos = sys.exc_info() try: - traceback.print_exception(*exc_infos, file=sys.stderr) + logger.exception('') self._context.hook_server_inspect_exception(event, None, exc_infos) finally: del exc_infos @@ -365,7 +390,7 @@ class Subscriber(Puller): def __init__(self, methods=None, context=None): super(Subscriber, self).__init__(methods=methods, context=context, zmq_socket=zmq.SUB) - self._events.setsockopt(zmq.SUBSCRIBE, '') + self._events.setsockopt(zmq.SUBSCRIBE, b'') def fork_task_context(functor, context=None): @@ -381,7 +406,7 @@ def fork_task_context(functor, context=None): - task1 is created to handle this event this task will be linked to the initial event context. zerorpc.Server does that for you. - task1 make use of some zerorpc.Client instances, the initial - event context is transfered on every call. + event context is transferred on every call. - task1 spawn a new task2. - task2 make use of some zerorpc.Client instances, it's a fresh @@ -390,7 +415,7 @@ def fork_task_context(functor, context=None): - task1 spawn a new fork_task_context(task3). - task3 make use of some zerorpc.Client instances, the initial - event context is transfered on every call. + event context is transferred on every call. A real use case is a distributed tracer. Each time a new event is created, a trace_id is injected in it or copied from the current task @@ -401,8 +426,9 @@ def fork_task_context(functor, context=None): - if the new task will make any zerorpc call, it should be wrapped. ''' context = context or Context.get_instance() - header = context.hook_get_task_context() + xheader = context.hook_get_task_context() + def wrapped(*args, **kargs): - context.hook_load_task_context(header) + context.hook_load_task_context(xheader) return functor(*args, **kargs) return wrapped diff --git a/zerorpc/decorators.py b/zerorpc/decorators.py index 3d20c71..43dfa64 100644 --- a/zerorpc/decorators.py +++ b/zerorpc/decorators.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -24,7 +24,7 @@ import inspect -from .patterns import * +from .patterns import ReqRep, ReqStream class DecoratorBase(object): @@ -33,7 +33,7 @@ class DecoratorBase(object): def __init__(self, functor): self._functor = functor self.__doc__ = functor.__doc__ - self.__name__ = functor.__name__ + self.__name__ = getattr(functor, "__name__", str(functor)) def __get__(self, instance, type_instance=None): if instance is None: diff --git a/zerorpc/events.py b/zerorpc/events.py index 7911151..f87d0b5 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,102 +23,161 @@ # SOFTWARE. +from __future__ import absolute_import +from builtins import str +from builtins import range + import msgpack import gevent.pool import gevent.queue import gevent.event import gevent.local -import gevent.coros - +import gevent.lock +import logging +import sys -import gevent_zmq as zmq +from . import gevent_zmq as zmq +from .exceptions import TimeoutExpired from .context import Context +from .channel_base import ChannelBase + + +if sys.version_info < (2, 7): + def get_pyzmq_frame_buffer(frame): + return frame.buffer[:] +else: + def get_pyzmq_frame_buffer(frame): + return frame.buffer + +# gevent <= 1.1.0.rc5 is missing the Python3 __next__ method. +if sys.version_info >= (3, 0) and gevent.version_info <= (1, 1, 0, 'rc', '5'): + setattr(gevent.queue.Channel, '__next__', gevent.queue.Channel.next) -class Sender(object): +logger = logging.getLogger(__name__) + + +class SequentialSender(object): def __init__(self, socket): self._socket = socket - self._send_queue = gevent.queue.Queue(maxsize=0) - self._send_task = gevent.spawn(self._sender) - def __del__(self): - self.close() + def _send(self, parts): + e = None + for i in range(len(parts) - 1): + try: + self._socket.send(parts[i], copy=False, flags=zmq.SNDMORE) + except (gevent.GreenletExit, gevent.Timeout) as e: + if i == 0: + raise + self._socket.send(parts[i], copy=False, flags=zmq.SNDMORE) + try: + self._socket.send(parts[-1], copy=False) + except (gevent.GreenletExit, gevent.Timeout) as e: + self._socket.send(parts[-1], copy=False) + if e: + raise e + + def __call__(self, parts, timeout=None): + if timeout: + with gevent.Timeout(timeout): + self._send(parts) + else: + self._send(parts) + + +class SequentialReceiver(object): + + def __init__(self, socket): + self._socket = socket + + def _recv(self): + e = None + parts = [] + while True: + try: + part = self._socket.recv(copy=False) + except (gevent.GreenletExit, gevent.Timeout) as e: + if len(parts) == 0: + raise + part = self._socket.recv(copy=False) + parts.append(part) + if not part.more: + break + if e: + raise e + return parts + + def __call__(self, timeout=None): + if timeout: + with gevent.Timeout(timeout): + return self._recv() + else: + return self._recv() + + +class Sender(SequentialSender): + + def __init__(self, socket): + self._socket = socket + self._send_queue = gevent.queue.Channel() + self._send_task = gevent.spawn(self._sender) def close(self): if self._send_task: self._send_task.kill() def _sender(self): - running = True for parts in self._send_queue: - for i in xrange(len(parts) - 1): - try: - self._socket.send(parts[i], flags=zmq.SNDMORE) - except gevent.GreenletExit: - if i == 0: - return - running = False - self._socket.send(parts[i], flags=zmq.SNDMORE) - self._socket.send(parts[-1]) - if not running: - return + super(Sender, self)._send(parts) - def __call__(self, parts): - self._send_queue.put(parts) + def __call__(self, parts, timeout=None): + try: + self._send_queue.put(parts, timeout=timeout) + except gevent.queue.Full: + raise TimeoutExpired(timeout) -class Receiver(object): +class Receiver(SequentialReceiver): def __init__(self, socket): self._socket = socket - self._recv_queue = gevent.queue.Queue(maxsize=0) + self._recv_queue = gevent.queue.Channel() self._recv_task = gevent.spawn(self._recver) - def __del__(self): - self.close() - def close(self): if self._recv_task: self._recv_task.kill() + self._recv_queue = None def _recver(self): - running = True while True: - parts = [] - while True: - try: - part = self._socket.recv() - except gevent.GreenletExit: - running = False - if len(parts) == 0: - return - part = self._socket.recv() - parts.append(part) - if not self._socket.getsockopt(zmq.RCVMORE): - break - if not running: - break + parts = super(Receiver, self)._recv() self._recv_queue.put(parts) - def __call__(self): - return self._recv_queue.get() + def __call__(self, timeout=None): + try: + return self._recv_queue.get(timeout=timeout) + except gevent.queue.Empty: + raise TimeoutExpired(timeout) class Event(object): - __slots__ = [ '_name', '_args', '_header' ] + __slots__ = ['_name', '_args', '_header', '_identity'] + # protocol details: + # - `name` and `header` keys must be unicode strings. + # - `message_id` and 'response_to' values are opaque bytes string. + # - `v' value is an integer. def __init__(self, name, args, context, header=None): self._name = name self._args = args if header is None: - self._header = { - 'message_id': context.new_msgid(), - 'v': 3 - } + self._header = {u'message_id': context.new_msgid(), u'v': 3} else: self._header = header + self._identity = None @property def header(self): @@ -136,12 +195,22 @@ def name(self, v): def args(self): return self._args + @property + def identity(self): + return self._identity + + @identity.setter + def identity(self, v): + self._identity = v + def pack(self): - return msgpack.Packer().pack((self._header, self._name, self._args)) + payload = (self._header, self._name, self._args) + r = msgpack.Packer(use_bin_type=True).pack(payload) + return r @staticmethod def unpack(blob): - unpacker = msgpack.Unpacker() + unpacker = msgpack.Unpacker(raw=False) unpacker.feed(blob) unpacked_msg = unpacker.unpack() @@ -164,43 +233,75 @@ def __str__(self, ignore_args=False): args = self._args try: args = '<<{0}>>'.format(str(self.unpack(self._args))) - except: + except Exception: pass - return '{0} {1} {2}'.format(self._name, self._header, - args) + if self._identity: + identity = ', '.join(repr(x.bytes) for x in self._identity) + return '<{0}> {1} {2} {3}'.format(identity, self._name, + self._header, args) + return '{0} {1} {2}'.format(self._name, self._header, args) -class Events(object): +class Events(ChannelBase): def __init__(self, zmq_socket_type, context=None): + self._debug = False self._zmq_socket_type = zmq_socket_type self._context = context or Context.get_instance() - self._socket = zmq.Socket(self._context, zmq_socket_type) - self._send = self._socket.send_multipart - self._recv = self._socket.recv_multipart - if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.XREQ, zmq.XREP): + self._socket = self._context.socket(zmq_socket_type) + + if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.DEALER, zmq.ROUTER): self._send = Sender(self._socket) - if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.XREQ, zmq.XREP): + elif zmq_socket_type in (zmq.REQ, zmq.REP): + self._send = SequentialSender(self._socket) + else: + self._send = None + + if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER): self._recv = Receiver(self._socket) + elif zmq_socket_type in (zmq.REQ, zmq.REP): + self._recv = SequentialReceiver(self._socket) + else: + self._recv = None @property - def recv_is_available(self): - return self._zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.XREQ, zmq.XREP) + def recv_is_supported(self): + return self._recv is not None + + @property + def emit_is_supported(self): + return self._send is not None def __del__(self): - if not self._socket.closed: - self.close() + try: + if not self._socket.closed: + self.close() + except (AttributeError, TypeError): + pass def close(self): try: self._send.close() - except AttributeError: + except (AttributeError, TypeError, gevent.GreenletExit): pass try: self._recv.close() - except AttributeError: + except (AttributeError, TypeError, gevent.GreenletExit): pass self._socket.close() + @property + def debug(self): + return self._debug + + @debug.setter + def debug(self, v): + if v != self._debug: + self._debug = v + if self._debug: + logger.debug('debug enabled') + else: + logger.debug('debug disabled') + def _resolve_endpoint(self, endpoint, resolve=True): if resolve: endpoint = self._context.hook_resolve_endpoint(endpoint) @@ -215,48 +316,56 @@ def connect(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.connect(endpoint_)) + logger.debug('connected to %s (status=%s)', endpoint_, r[-1]) return r def bind(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.bind(endpoint_)) + logger.debug('bound to %s (status=%s)', endpoint_, r[-1]) return r - def create_event(self, name, args, xheader={}): + def disconnect(self, endpoint, resolve=True): + r = [] + for endpoint_ in self._resolve_endpoint(endpoint, resolve): + r.append(self._socket.disconnect(endpoint_)) + logger.debug('disconnected from %s (status=%s)', endpoint_, r[-1]) + return r + + def new_event(self, name, args, xheader=None): event = Event(name, args, context=self._context) - for k, v in xheader.items(): - if k == 'zmqid': - continue - event.header[k] = v + if xheader: + event.header.update(xheader) return event - def emit_event(self, event, identity=None): - if identity is not None: - parts = list(identity) - parts.extend(['', event.pack()]) - elif self._zmq_socket_type in (zmq.XREQ, zmq.XREP): - parts = ('', event.pack()) + def emit_event(self, event, timeout=None): + if self._debug: + logger.debug('--> %s', event) + if event.identity: + parts = list(event.identity or list()) + parts.extend([b'', event.pack()]) + elif self._zmq_socket_type in (zmq.DEALER, zmq.ROUTER): + parts = (b'', event.pack()) else: parts = (event.pack(),) - self._send(parts) - - def emit(self, name, args, xheader={}): - event = self.create_event(name, args, xheader) - identity = xheader.get('zmqid', None) - return self.emit_event(event, identity) + self._send(parts, timeout) - def recv(self): - parts = self._recv() - if len(parts) == 1: - identity = None - blob = parts[0] - else: + def recv(self, timeout=None): + parts = self._recv(timeout=timeout) + if len(parts) > 2: identity = parts[0:-2] blob = parts[-1] - event = Event.unpack(blob) - if identity is not None: - event.header['zmqid'] = identity + elif len(parts) == 2: + identity = parts[0:-1] + blob = parts[-1] + else: + identity = None + blob = parts[0] + event = Event.unpack(get_pyzmq_frame_buffer(blob)) + event.identity = identity + if self._debug: + logger.debug('<-- %s', event) return event def setsockopt(self, *args): @@ -265,39 +374,3 @@ def setsockopt(self, *args): @property def context(self): return self._context - - -class WrappedEvents(object): - - def __init__(self, channel): - self._channel = channel - - def close(self): - pass - - @property - def recv_is_available(self): - return self._channel.recv_is_available - - def create_event(self, name, args, xheader={}): - event = Event(name, args, self._channel.context) - event.header.update(xheader) - return event - - def emit_event(self, event, identity=None): - event_payload = (event.header, event.name, event.args) - wrapper_event = self._channel.create_event('w', event_payload) - self._channel.emit_event(wrapper_event) - - def emit(self, name, args, xheader={}): - wrapper_event = self.create_event(name, args, xheader) - self.emit_event(wrapper_event) - - def recv(self, timeout=None): - wrapper_event = self._channel.recv() - (header, name, args) = wrapper_event.args - return Event(name, args, None, header) - - @property - def context(self): - return self._channel.context diff --git a/zerorpc/exceptions.py b/zerorpc/exceptions.py index 781200e..14a1419 100644 --- a/zerorpc/exceptions.py +++ b/zerorpc/exceptions.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index 017b1f5..54420ae 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -25,14 +25,28 @@ # Based on https://github.com/traviscline/gevent-zeromq/ # We want to act like zmq -from zmq import * +from zmq import * # noqa + +try: + # Try to import enums for pyzmq >= 23.0.0 + from zmq.constants import * # noqa +except ImportError: + pass + + +# Explicit import to please flake8 +from zmq import ZMQError # A way to access original zmq import zmq as _zmq import gevent.event import gevent.core -import sys +import errno +from logging import getLogger + +logger = getLogger(__name__) + class Context(_zmq.Context): @@ -47,17 +61,21 @@ class Socket(_zmq.Socket): def __init__(self, context, socket_type): super(Socket, self).__init__(context, socket_type) on_state_changed_fd = self.getsockopt(_zmq.FD) - self._readable = gevent.event.Event() - self._writable = gevent.event.Event() + # NOTE: pyzmq 13.0.0 messed up with setattr (they turned it into a + # non-op) and you can't assign attributes normally anymore, hence the + # tricks with self.__dict__ here + self.__dict__["_readable"] = gevent.event.Event() + self.__dict__["_writable"] = gevent.event.Event() try: # gevent>=1.0 - self._state_event = gevent.hub.get_hub().loop.io( + self.__dict__["_state_event"] = gevent.hub.get_hub().loop.io( on_state_changed_fd, gevent.core.READ) self._state_event.start(self._on_state_changed) except AttributeError: # gevent<1.0 - self._state_event = gevent.core.read_event(on_state_changed_fd, - self._on_state_changed, persist=True) + self.__dict__["_state_event"] = \ + gevent.core.read_event(on_state_changed_fd, + self._on_state_changed, persist=True) def _on_state_changed(self, event=None, _evtype=None): if self.closed: @@ -65,7 +83,14 @@ def _on_state_changed(self, event=None, _evtype=None): self._readable.set() return - events = self.getsockopt(_zmq.EVENTS) + while True: + try: + events = self.getsockopt(_zmq.EVENTS) + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise + if events & _zmq.POLLOUT: self._writable.set() if events & _zmq.POLLIN: @@ -81,6 +106,14 @@ def close(self): self._state_event.cancel() super(Socket, self).close() + def connect(self, *args, **kwargs): + while True: + try: + return super(Socket, self).connect(*args, **kwargs) + except _zmq.ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise + def send(self, data, flags=0, copy=True, track=False): if flags & _zmq.NOBLOCK: return super(Socket, self).send(data, flags, copy, track) @@ -99,8 +132,8 @@ def send(self, data, flags=0, copy=True, track=False): # send and recv on the socket. self._on_state_changed() return msg - except _zmq.ZMQError, e: - if e.errno != _zmq.EAGAIN: + except _zmq.ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._writable.clear() # The following sleep(0) force gevent to switch out to another @@ -112,10 +145,14 @@ def send(self, data, flags=0, copy=True, track=False): # the same CPU load, you get a better throughput (roughly 18.75%). gevent.sleep(0) while not self._writable.wait(timeout=1): - if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: - print>>sys.stderr, "/!\\ gevent_zeromq BUG /!\\ " \ - "catching up after missing event (SEND) /!\\" - break + try: + if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: + logger.error("/!\\ gevent_zeromq BUG /!\\ " + "catching up after missing event (SEND) /!\\") + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise def recv(self, flags=0, copy=True, track=False): if flags & _zmq.NOBLOCK: @@ -135,8 +172,8 @@ def recv(self, flags=0, copy=True, track=False): # send and recv on the socket. self._on_state_changed() return msg - except _zmq.ZMQError, e: - if e.errno != _zmq.EAGAIN: + except _zmq.ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._readable.clear() # The following sleep(0) force gevent to switch out to another @@ -148,7 +185,11 @@ def recv(self, flags=0, copy=True, track=False): # the same CPU load, you get a better throughput (roughly 18.75%). gevent.sleep(0) while not self._readable.wait(timeout=1): - if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: - print>>sys.stderr, "/!\\ gevent_zeromq BUG /!\\ " \ - "catching up after missing event (RECV) /!\\" - break + try: + if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: + logger.error("/!\\ gevent_zeromq BUG /!\\ " + "catching up after missing event (RECV) /!\\") + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index e7af22f..23b974d 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -28,17 +28,19 @@ import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock -from .exceptions import * +from .exceptions import LostRemote, TimeoutExpired +from .channel_base import ChannelBase -class HeartBeatOnChannel(object): +class HeartBeatOnChannel(ChannelBase): def __init__(self, channel, freq=5, passive=False): + self._closed = False self._channel = channel self._heartbeat_freq = freq - self._input_queue = gevent.queue.Queue(maxsize=0) + self._input_queue = gevent.queue.Channel() self._remote_last_hb = None self._lost_remote = False self._recv_task = gevent.spawn(self._recver) @@ -49,13 +51,15 @@ def __init__(self, channel, freq=5, passive=False): self._start_heartbeat() @property - def recv_is_available(self): - return self._channel.recv_is_available + def recv_is_supported(self): + return self._channel.recv_is_supported - def __del__(self): - self.close() + @property + def emit_is_supported(self): + return self._channel.emit_is_supported def close(self): + self._closed = True if self._heartbeat_task is not None: self._heartbeat_task.kill() self._heartbeat_task = None @@ -73,25 +77,26 @@ def _heartbeat(self): self._remote_last_hb = time.time() if time.time() > self._remote_last_hb + self._heartbeat_freq * 2: self._lost_remote = True - gevent.kill(self._parent_coroutine, - self._lost_remote_exception()) + if not self._closed: + gevent.kill(self._parent_coroutine, + self._lost_remote_exception()) break - self._channel.emit('_zpc_hb', (0,)) # 0 -> compat with protocol v2 + self._channel.emit(u'_zpc_hb', (0,)) # 0 -> compat with protocol v2 def _start_heartbeat(self): - if self._heartbeat_task is None and self._heartbeat_freq is not None: + if self._heartbeat_task is None and self._heartbeat_freq is not None and not self._closed: self._heartbeat_task = gevent.spawn(self._heartbeat) def _recver(self): while True: event = self._channel.recv() if self._compat_v2 is None: - self._compat_v2 = event.header.get('v', 0) < 3 - if event.name == '_zpc_hb': + self._compat_v2 = event.header.get(u'v', 0) < 3 + if event.name == u'_zpc_hb': self._remote_last_hb = time.time() self._start_heartbeat() if self._compat_v2: - event.name = '_zpc_more' + event.name = u'_zpc_more' self._input_queue.put(event) else: self._input_queue.put(event) @@ -100,31 +105,24 @@ def _lost_remote_exception(self): return LostRemote('Lost remote after {0}s heartbeat'.format( self._heartbeat_freq * 2)) - def create_event(self, name, args, xheader={}): - if self._compat_v2 and name == '_zpc_more': - name = '_zpc_hb' - return self._channel.create_event(name, args, xheader) + def new_event(self, name, args, header=None): + if self._compat_v2 and name == u'_zpc_more': + name = u'_zpc_hb' + return self._channel.new_event(name, args, header) - def emit_event(self, event): + def emit_event(self, event, timeout=None): if self._lost_remote: raise self._lost_remote_exception() - self._channel.emit_event(event) - - def emit(self, name, args, xheader={}): - event = self.create_event(name, args, xheader) - self.emit_event(event) + self._channel.emit_event(event, timeout) def recv(self, timeout=None): if self._lost_remote: raise self._lost_remote_exception() - try: - event = self._input_queue.get(timeout=timeout) + return self._input_queue.get(timeout=timeout) except gevent.queue.Empty: raise TimeoutExpired(timeout) - return event - @property def channel(self): return self._channel diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index 526356c..3623e17 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,68 +23,71 @@ # SOFTWARE. -class ReqRep: +class ReqRep(object): - def process_call(self, context, bufchan, req_event, functor): + def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) result = functor(*req_event.args) - rep_event = bufchan.create_event('OK', (result,), + rep_event = channel.new_event(u'OK', (result,), context.hook_get_task_context()) context.hook_server_after_exec(req_event, rep_event) - bufchan.emit_event(rep_event) + channel.emit_event(rep_event) def accept_answer(self, event): - return True + return event.name in (u'OK', u'ERR') - def process_answer(self, context, bufchan, req_event, rep_event, + def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): - if rep_event.name == 'ERR': - exception = handle_remote_error(rep_event) - context.hook_client_after_request(req_event, rep_event, exception) - raise exception - context.hook_client_after_request(req_event, rep_event) - bufchan.close() - result = rep_event.args[0] - return result + try: + if rep_event.name == u'ERR': + exception = handle_remote_error(rep_event) + context.hook_client_after_request(req_event, rep_event, exception) + raise exception + context.hook_client_after_request(req_event, rep_event) + return rep_event.args[0] + finally: + channel.close() -class ReqStream: +class ReqStream(object): - def process_call(self, context, bufchan, req_event, functor): + def process_call(self, context, channel, req_event, functor): context.hook_server_before_exec(req_event) xheader = context.hook_get_task_context() for result in iter(functor(*req_event.args)): - bufchan.emit('STREAM', result, xheader) - done_event = bufchan.create_event('STREAM_DONE', None, xheader) + channel.emit(u'STREAM', result, xheader) + done_event = channel.new_event(u'STREAM_DONE', None, xheader) # NOTE: "We" made the choice to call the hook once the stream is done, - # the other choice was to call it at each iteration. I don't think that - # one choice is better than the other, so I'm fine with changing this + # the other choice was to call it at each iteration. I donu't think that + # one choice is better than the other, so Iu'm fine with changing this # or adding the server_after_iteration and client_after_iteration hooks. context.hook_server_after_exec(req_event, done_event) - bufchan.emit_event(done_event) + channel.emit_event(done_event) def accept_answer(self, event): - return event.name in ('STREAM', 'STREAM_DONE') + return event.name in (u'STREAM', u'STREAM_DONE') - def process_answer(self, context, bufchan, req_event, rep_event, + def process_answer(self, context, channel, req_event, rep_event, handle_remote_error): def is_stream_done(rep_event): - return rep_event.name == 'STREAM_DONE' - bufchan.on_close_if = is_stream_done + return rep_event.name == u'STREAM_DONE' + channel.on_close_if = is_stream_done def iterator(req_event, rep_event): - while rep_event.name == 'STREAM': - # Like in process_call, we made the choice to call the - # after_exec hook only when the stream is done. - yield rep_event.args - rep_event = bufchan.recv() - if rep_event.name == 'ERR': - exception = handle_remote_error(rep_event) - context.hook_client_after_request(req_event, rep_event, exception) - raise exception - context.hook_client_after_request(req_event, rep_event) - bufchan.close() + try: + while rep_event.name == u'STREAM': + # Like in process_call, we made the choice to call the + # after_exec hook only when the stream is done. + yield rep_event.args + rep_event = channel.recv() + if rep_event.name == u'ERR': + exception = handle_remote_error(rep_event) + context.hook_client_after_request(req_event, rep_event, exception) + raise exception + context.hook_client_after_request(req_event, rep_event) + finally: + channel.close() return iterator(req_event, rep_event) diff --git a/zerorpc/socket.py b/zerorpc/socket.py index 2a7020e..35cb7e4 100644 --- a/zerorpc/socket.py +++ b/zerorpc/socket.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -41,3 +41,14 @@ def connect(self, endpoint, resolve=True): def bind(self, endpoint, resolve=True): return self._events.bind(endpoint, resolve) + + def disconnect(self, endpoint, resolve=True): + return self._events.disconnect(endpoint, resolve) + + @property + def debug(self): + return self._events.debug + + @debug.setter + def debug(self, v): + self._events.debug = v diff --git a/zerorpc/version.py b/zerorpc/version.py index a6db92a..4324a5a 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.4.1' -__author__ = 'dotCloud, Inc.' +__version__ = '0.6.3' +__author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' -__copyright__ = 'Copyright 2012 dotCloud, Inc.' +__copyright__ = 'Copyright 2015 François-Xavier Bourlet .'