diff --git a/.gitignore b/.gitignore index 58311d7..8aa21a8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ *.egg *.egg-info .tox +.idea diff --git a/.travis.yml b/.travis.yml index 2a1a60c..67c2d20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,32 @@ +dist: trusty +sudo: false language: python python: + - 2.6 - 2.7 -script: nosetests -before_install: - - sudo apt-get update - - sudo apt-get install python-dev libevent-dev + - 3.4 + - 3.5 +env: + matrix: + - PYZMQ='pyzmq>=15' + - PYZMQ='pyzmq>=14,<15' + - PYZMQ='pyzmq>=13,<14' +matrix: + fast_finish: true +addons: + apt: + packages: + - python-dev + - libevent-dev install: - - pip install nose --use-mirrors - - pip install . --use-mirrors + - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then + pip install flake8; + fi + - "pip install nose $PYZMQ" + - pip install . + - pip freeze +script: + - if [ $TRAVIS_PYTHON_VERSION != '2.6' ]; then + flake8 --ignore=E501,E128 zerorpc bin; + fi + - pytest -v diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 6e329ee..0000000 --- a/AUTHORS +++ /dev/null @@ -1,6 +0,0 @@ -Andrea Luzzardi -François-Xavier Bourlet -Jérôme Petazzoni -Samuel Alba -Solomon Hykes -Sébastien Pahl diff --git a/LICENSE b/LICENSE index 43fb7a1..49bbb6e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ Open Source Initiative OSI - The MIT License (MIT):Licensing The MIT License (MIT) -Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.rst b/README.rst index 784382a..3538015 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,11 @@ zerorpc ======= -.. image:: https://secure.travis-ci.org/dotcloud/zerorpc-python.png - :target: http://travis-ci.org/dotcloud/zerorpc-python +.. image:: https://travis-ci.org/0rpc/zerorpc-python.svg?branch=master + :target: https://travis-ci.org/0rpc/zerorpc-python + +Mailing list: zerorpc@googlegroups.com (https://groups.google.com/d/forum/zerorpc) + zerorpc is a flexible RPC implementation based on zeromq and messagepack. Service APIs exposed with zerorpc are called "zeroservices". @@ -13,6 +16,14 @@ with a convenient script, "zerorpc", allowing to: * expose Python modules without modifying a single line of code, * call those modules remotely through the command line. +Installation +------------ + +On most systems, its a matter of:: + + $ pip install zerorpc + +Depending of the support from Gevent and PyZMQ on your system, you might need to install `libev` (for gevent) and `libzmq` (for pyzmq) with the development files. Create a server with a one-liner -------------------------------- @@ -25,7 +36,7 @@ we will expose the Python "time" module:: .. note:: The bind address uses the zeromq address format. You are not limited to TCP transport: you could as well specify ipc:///tmp/time to use - host-local sockets, for instance. "tcp://*:1234" is a short-hand to + host-local sockets, for instance. "tcp://\*:1234" is a short-hand to "tcp://0.0.0.0:1234" and means "listen on TCP port 1234, accepting connections on all IP addresses". @@ -35,22 +46,22 @@ Call the server from the command-line Now, in another terminal, call the exposed module:: - $ zerorpc --client --connect tcp://*:1234 strftime %Y/%m/%d - Connecting to "tcp://*:1234" + $ zerorpc --client --connect tcp://127.0.0.1:1234 strftime %Y/%m/%d + Connecting to "tcp://127.0.0.1:1234" "2011/03/07" Since the client usecase is the most common one, "--client" is the default parameter, and you can remove it safely:: - $ zerorpc --connect tcp://*:1234 strftime %Y/%m/%d - Connecting to "tcp://*:1234" + $ zerorpc --connect tcp://127.0.0.1:1234 strftime %Y/%m/%d + Connecting to "tcp://127.0.0.1:1234" "2011/03/07" Moreover, since the most common usecase is to *connect* (as opposed to *bind*) you can also omit "--connect":: - $ zerorpc tcp://*:1234 strftime %Y/%m/%d - Connecting to "tcp://*:1234" + $ zerorpc tcp://127.0.0.1:1234 strftime %Y/%m/%d + Connecting to "tcp://127.0.0.1:1234" "2011/03/07" @@ -60,8 +71,8 @@ See remote service documentation You can introspect the remote service; it happens automatically if you don't specify the name of the function you want to call:: - $ zerorpc tcp://*:1234 - Connecting to "tcp://*:1234" + $ zerorpc tcp://127.0.0.1:1234 + Connecting to "tcp://127.0.0.1:1234" tzset tzset(zone) ctime ctime(seconds) -> string clock clock() -> floating point number @@ -82,8 +93,8 @@ Specifying non-string arguments Now, see what happens if we try to call a function expecting a non-string argument:: - $ zerorpc tcp://*:1234 sleep 3 - Connecting to "tcp://*:1234" + $ zerorpc tcp://127.0.0.1:1234 sleep 3 + Connecting to "tcp://127.0.0.1:1234" Traceback (most recent call last): [...] TypeError: a float is required @@ -91,8 +102,8 @@ argument:: That's because all command-line arguments are handled as strings. Don't worry, we can specify any kind of argument using JSON encoding:: - $ zerorpc --json tcp://*:1234 sleep 3 - Connecting to "tcp://*:1234" + $ zerorpc --json tcp://127.0.0.1:1234 sleep 3 + Connecting to "tcp://127.0.0.1:1234" [wait for 3 seconds...] null @@ -105,15 +116,15 @@ your server to act as a kind of worker, and connect to a hub or queue which will dispatch requests. You can achieve this by swapping "--bind" and "--connect":: - $ zerorpc --bind tcp://*:1234 localtime + $ zerorpc --bind tcp://*:1234 strftime %Y/%m/%d -We now have "something" wanting to call the "localtime" function, and waiting +We now have "something" wanting to call the "strftime" function, and waiting for a worker to connect to it. Let's start the worker:: - $ zerorpc --server tcp://*:1234 time + $ zerorpc --server tcp://127.0.0.1:1234 time The worker will connect to the listening client and ask him "what should I -do?"; the client will send the "localtime" function call; the worker will +do?"; the client will send the "strftime" function call; the worker will execute it and return the result. The first program will display the local time and exit. The worker will remain running. @@ -126,7 +137,7 @@ the "--bind" option:: $ zerorpc --server --bind tcp://*:1234 --bind ipc:///tmp/time time -You can then connect to it using either "zerorpc tcp://*:1234" or +You can then connect to it using either "zerorpc tcp://\*:1234" or "zerorpc ipc:///tmp/time". Wait, there is more! You can even mix "--bind" and "--connect". That means @@ -153,7 +164,7 @@ python API. Below are a few examples. Here's how to expose an object of your choice as a zeroservice:: - class Cooler: + class Cooler(object): """ Various convenience methods to make things cooler. """ def add_man(self, sentence): @@ -183,11 +194,11 @@ Let's save this code to *cooler.py* and run it:: Now, in another terminal, let's try connecting to our awesome zeroservice:: - $ zerorpc -j tcp://:4242 add_42 1 + $ zerorpc -j tcp://localhost:4242 add_42 1 43 - $ zerorpc tcp://:4242 add_man 'I own a mint-condition Wolkswagen Golf' - "I own a mint-condition Wolkswagen Gold, man!" - $ zerorpc tcp://:4242 boat 'I own a mint-condition Wolkswagen Gold, man!' + $ zerorpc tcp://localhost:4242 add_man 'I own a mint-condition Volkswagen Golf' + "I own a mint-condition Volkswagen Golf, man!" + $ zerorpc tcp://localhost:4242 boat 'I own a mint-condition Volkswagen Golf, man!' "I'm on a boat!" diff --git a/bin/zerorpc b/bin/zerorpc index a318577..c9d31d3 100755 --- a/bin/zerorpc +++ b/bin/zerorpc @@ -3,7 +3,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,213 +23,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - -import argparse -import json -import sys -import inspect import os -from pprint import pprint - -import zerorpc - - -parser = argparse.ArgumentParser( - description='Make a zerorpc call to a remote service.' - ) - -client_or_server = parser.add_mutually_exclusive_group() -client_or_server.add_argument('--client', action='store_true', default=True, - help='remote procedure call mode (default)') -client_or_server.add_argument('--server', action='store_false', dest='client', - help='turn a given python module into a server') - -parser.add_argument('--connect', action='append', metavar='address', - help='specify address to connect to. Can be specified \ - multiple times and in conjunction with --bind') -parser.add_argument('--bind', action='append', metavar='address', - help='specify address to listen to. Can be specified \ - multiple times and in conjunction with --connect') -parser.add_argument('--timeout', default=30, metavar='seconds', type=int, - help='abort request after X seconds. \ - (default: 30s, --client only)') -parser.add_argument('--heartbeat', default=5, metavar='seconds', type=int, - help='heartbeat frequency. You should always use \ - the same frequency as the server. (default: 5s)') -parser.add_argument('-j', '--json', default=False, action='store_true', - help='arguments are in JSON format and will be be parsed \ - before being sent to the remote') -parser.add_argument('-pj', '--print-json', default=False, action='store_true', - help='print result in JSON format.') -parser.add_argument('-?', '--inspect', default=False, action='store_true', - help='retrieve detailed informations for the given \ - remote (cf: command) method. If not method, display \ - a list of remote methods signature. (only for --client).') -parser.add_argument('--active-hb', default=False, action='store_true', - help='enable active heartbeat. The default is to \ - wait for the server to send the first heartbeat') -parser.add_argument('address', nargs='?', help='address to connect to. Skip \ - this if you specified --connect or --bind at least once') -parser.add_argument('command', nargs='?', - help='remote procedure to call if --client (default) or \ - python module/class to load if --server. If no command is \ - specified, a list of remote methods are displayed.') -parser.add_argument('params', nargs='*', - help='parameters for the remote call if --client \ - (default)') - - -def setup_links(args, socket): - if args.bind: - for endpoint in args.bind: - print 'binding to "{0}"'.format(endpoint) - socket.bind(endpoint) - addresses = [] - if args.address: - addresses.append(args.address) - if args.connect: - addresses.extend(args.connect) - for endpoint in addresses: - print 'connecting to "{0}"'.format(endpoint) - socket.connect(endpoint) - - -def run_server(args): - server_obj_path = args.command - - sys.path.insert(0, os.getcwd()) - if '.' in server_obj_path: - modulepath, objname = server_obj_path.rsplit('.', 1) - module = __import__(modulepath, fromlist=[objname]) - server_obj = getattr(module, objname) - else: - server_obj = __import__(server_obj_path) - - if callable(server_obj): - server_obj = server_obj() - - server = zerorpc.Server(server_obj, heartbeat=args.heartbeat) - setup_links(args, server) - print 'serving "{0}"'.format(server_obj_path) - return server.run() - - -# this function does a really intricate job to keep backward compatibility -# with a previous version of zerorpc, and lazily retrieving results if possible -def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): - try: - remote_detailled_methods = client._zerorpc_inspect(method, - long_doc)['methods'] - - if include_argspec: - r = [(name + (inspect.formatargspec(*argspec) if argspec else - '(...)'), doc if doc else '') - for name, argspec, doc in remote_detailled_methods] - else: - r = [(name, doc if doc else '') - for name, argspec, doc in remote_detailled_methods] - - longest_name_len = max(len(name) for name, doc in r) - return (longest_name_len, r) - except (zerorpc.RemoteError, NameError): - pass - - if method is None: - remote_methods = client._zerorpc_list() - else: - remote_methods = [method] - - def remote_detailled_methods(): - for name in remote_methods: - if include_argspec: - argspec = client._zerorpc_args(name) - else: - argspec = None - docstring = client._zerorpc_help(name) - if docstring and not long_doc: - docstring = docstring.split('\n', 1)[0] - yield (name, argspec, docstring if docstring else '') - - if not include_argspec: - longest_name_len = max(len(name) for name in remote_methods) - return (longest_name_len, ((name, doc) for name, argspec, doc in - remote_detailled_methods())) - - r = [(name + (inspect.formatargspec(*argspec) - if argspec else '(...)'), doc) - for name, argspec, doc in remote_detailled_methods()] - longest_name_len = max(len(name) for name, doc in r) - return (longest_name_len, r) - - -def run_client(args): - client = zerorpc.Client(timeout=args.timeout, heartbeat=args.heartbeat, - passive_heartbeat=not args.active_hb) - setup_links(args, client) - if not args.command: - (longest_name_len, detailled_methods) = zerorpc_inspect(client, - long_doc=False, include_argspec=args.inspect) - if args.inspect: - for (name, doc) in detailled_methods: - print name - else: - for (name, doc) in detailled_methods: - print '{0} {1}'.format(name.ljust(longest_name_len), doc) - return - if args.inspect: - (longest_name_len, detailled_methods) = zerorpc_inspect(client, - method=args.command) - (name, doc) = detailled_methods[0] - print '\n{0}\n\n{1}\n'.format(name, doc) - return - if args.json: - call_args = [json.loads(x) for x in args.params] - else: - call_args = args.params - results = client(args.command, *call_args) - if getattr(results, 'next', None) is None: - if args.print_json: - json.dump(results, sys.stdout) - else: - pprint(results) - else: - # streaming responses - if args.print_json: - first = True - sys.stdout.write('[') - for result in results: - if first: - first = False - else: - sys.stdout.write(',') - json.dump(result, sys.stdout) - sys.stdout.write(']') - else: - for result in results: - pprint(result) - - -def main(): - args = parser.parse_args() - - if args.bind or args.connect: - if args.command: - args.params.insert(0, args.command) - args.command = args.address - args.address = None - - if not (args.bind or args.connect or args.address): - parser.print_help() - return -1 - - if args.client: - return run_client(args) - - if not args.command: - parser.print_help() - return -1 +import sys +sys.path.append(os.path.dirname(os.path.dirname(sys.argv[0]))) - return run_server(args) +from zerorpc import cli # NOQA -if __name__ == '__main__': - exit(main()) +if __name__ == "__main__": + exit(cli.main()) diff --git a/doc/protocol.md b/doc/protocol.md new file mode 100644 index 0000000..cfc83d2 --- /dev/null +++ b/doc/protocol.md @@ -0,0 +1,248 @@ +# zerorpc Protocol + +THIS DOCUMENT IS INCOMPLETE, WORK IN PROGRESS! + +This document attempts to define the zerorpc protocol. + +## Introduction & History + +zerorpc features a dead-simple API for exposing any object or module over the +network, and a powerful gevent implementation which supports multiple ZMQ +socket types, streaming, heartbeats and more. It also includes a simple +command-line tool for interactive querying and introspection. + +zerorpc uses ZMQ as a transport, but uses a communication protocol that is +transport-agnostic. For a long time the reference documentation for that +protocol was the python code itself. However since its recent surge in +popularity many people have expressed interest in porting it to other +programming languages. We hope that this standalone protocol documentation will +help them. + +> The python implementation of zerorpc act as a reference for the whole +> protocol. New features and experiments are implemented and tested in this +> version first. + +[zerorpc](http://www.zerorpc.io) is a modern communication layer for +distributed systems built on top of [ZeroMQ](http://zeromq.org), initially +developed at [dotCloud](http://www.dotcloud.com) starting in 2010 and +open-sourced in 2012. when dotCloud pivoted to [Docker](http://www.docker.com), +dotCloud was acquired by [cloudControl](https://www.cloudcontrol.com/), which +then migrated over to theirs own PaaS before shutting it down. + +In 2015, I (François-Xavier Bourlet) was given zerorpc by cloudControl, in an +attempt to revive the project, maintain it, and hopefully drive its +development. + +### Warning + +A short warning: zerorpc started as a simple tool to solve a simple problem. It +was progressively refined and improved to satisfy the growing needs of the +dotCloud platform. The emphasis is on practicality, robustness and operational +simplicity - sometimes at the expense of formal elegance and optimizations. We +will gladly welcome patches focused on the latter so long as they don't hurt +the former. + + +## Layers + +Before diving into any details, let's divide zerorpc's protocol in three +different layers: + + 1. Wire (or transport) layer; a combination of ZMQ + and msgpack . + 2. Event (or message) layer; this is probably the most complex part, since + it handles heartbeat, multiplexing, and events. + 3. RPC layer; that's where you can find the notion of request and response. + +## Wire layer + +The wire layer is a combination of ZMQ and msgpack. + +The basics: + + - A zerorpc server can listen on as many ZMQ sockets as you like. Actually, + a ZMQ socket can bind to multiple addresses. It can even *connect* to the + clients (think about it as a worker connecting to a hub), but there are + some limitations in that case (see below). zerorpc doesn't + have to do anything specific for that: ZMQ handles it automatically. + - A zerorpc client can connect to multiple zerorpc servers. However, it should + create a new ZMQ socket for each connection. + +Since zerorpc implements heartbeat and streaming, it expects a kind of +persistent, end-to-end, connection between the client and the server. +It means that we cannot use the load-balancing features built into ZMQ. +Otherwise, the various messages composing a single conversation could +end up in different places. + +That's why there are limitations when the server connects to the client: +if there are multiple servers connecting to the same client, bad things +will happen. + +> Note that the current implementation of zerorpc for Python doesn't implement +> its own load-balancing (yet), and still uses one ZMQ socket for connecting to +> many servers. You can still use ZMQ load-balancing if you accept to disable +> heartbeat and don't use streamed responses. + +Every event from the event layer will be serialized with msgpack. + + +## Event layer + +The event layer is the most complex of all three layers. The majority of the +code for the Python implementation deals with this layer. + +This layer provides: + + - basic events; + - multiplexed channels, allowing concurrency. + + +### Basic Events + +An event is a tuple (or array in JSON), containing in the following order: + + 1. the event's header -> dictionary (or object in JSON) + 2. the event's names -> string + 3. the event's arguments -> any kind of value; but in practice, for backward + compatibility, it is recommended that this is a tuple (an empty one is OK). + +All events headers must contain an unique message id and the protocol version: + + { + "message_id": "6ce9503a-bfb8-486a-ac79-e2ed225ace79", + "v": 3 + } + +The message id should be unique for the lifetime of the connection between a +client and a server. + +> It doesn't need to be an UUID, but again, for backward compatibility reasons, +> it is better if it follows the UUID format. + +This document talks only about the version 3 of the protocol. + +> The Python implementation has a lot of backward compatibility code, to handle +> communication between all three versions of the protocol. + + +### Multiplexed Channels + + - Each new event opens a new channel implicitly. + - The id of the new event will represent the channel id for the connection. + - Each consecutive event on a channel will have the header field "response_to" + set to the channel id: + + { + "message_id": "6ce9503a-bfb8-486a-ac79-e2ed225ace79", + "response_to": "6636fb60-2bca-4ecb-8cb4-bbaaf174e9e6" + } + +#### Heartbeat + +Each part of a channel must send a heartbeat at regular intervals. + +The default heartbeat frequency is 5 seconds. + +> Note that technically, the heartbeat could be sitting on the connection level +> instead of the channel level; but again, backward compatibility requires +> to run it per channel at this point. + +The heartbeat is defined as follow: + + - Event's name: '\_zpc\_hb' + - Event's args: null + +When no heartbeat even is received after 2 heartbeat intervals (so, 10s by default), +we consider that the remote is lost. + +> The Python implementation raises the LostRemote exception, and even +> manages to cancel a long-running task on a LostRemote. FIXME what does that mean? + +#### Buffering (or congestion control) on channels + +Both sides have a buffer for incoming messages on a channel. A peer can +send an advisory message to the other end of the channel, to inform it of the +size of its local buffer. This is a hint for the remote, to tell it "send me +more data!" + + - Event's name: '\_zpc\_more' + - Event's args: integer representing how many entries are available in the client's buffer. + +FIXME WIP + +## RPC Layer + +In the first version of zerorpc, this was the main (and only) layer. +Three kinds of events can occur at this layer: request (=function call), +response (=function return), error (=exception). + +Request: + + - Event's name: string with the name of the method to call. + - Event's args: tuple of arguments for the method. + +Note: keyword arguments are not supported, because some languages don't +support them. If you absolutely want to call functions with keyword +arguments, you can use a wrapper; e.g. expose a function like +"call_with_kwargs(function_name, args, kwargs)", where args is a list, +and kwargs a dict. It might be an interesting idea to add such a +helper function into zerorpc default methods (see below for definitions +of existing default methods). + +Response: + + - Event's name: string "OK" + - Event's args: tuple containing the returned value + +> Note that if the return value is a tuple, it is itself wrapped inside a +> tuple - again, for backward compatibility reasons. + +FIXME - is [] equivalent to [null]? + +If an error occurs (either at the transport level, or if an uncaught +exception is raised), we use the ERR event. + + - Event's name: string "ERR" + - Event's args: tuple of 3 strings: + - Name of the error (it should be the exception class name, or another + meaningful keyword). + - Human representation of the error (preferably in english). + - If possible a pretty printed traceback of the call stack when the error occurred. + +> A future version of the protocol will probably add a structured version of the +> traceback, allowing machine-to-machine stack walking and better cross-language +> exception representation. + + +### Default calls + +When exposing some code with zerorpc, a number of methods/functions are +automatically added, to provide useful debugging and development tools. + + - \_zerorpc\_ping() just answers with a pong message. + - \_zerorpc\_inspect() returns all the available calls, with their + signature and documentation. + +FIXME we should rather standardize about the basic introspection calls. + + +### Streaming + +At the protocol level, streaming is straightforward. When a server wants +to stream some data, instead of sending a "OK" message, it sends a "STREAM" +message. The client will know that it needs to keep waiting for more. +At the end of the stream, a "STREAM_DONE" message is expected. + +Formal definitions follow. + +Messages part of a stream: + + - Event's name: string "STREAM" + - Event's args: tuple containing the streamed value + +When the STREAM reaches its end: + + - Event's name: string "STREAM\_DONE" + - Event's args: null + +> The Python implementation represents a stream by an iterator on both sides. diff --git a/setup.py b/setup.py index 3f5311f..b07ebcb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,7 +22,12 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -execfile('zerorpc/version.py') +import sys + +if sys.version_info < (3, 0): + execfile('zerorpc/version.py') +else: + exec(compile(open('zerorpc/version.py', encoding='utf8').read(), 'zerorpc/version.py', 'exec')) try: from setuptools import setup @@ -30,25 +35,39 @@ from distutils.core import setup +requirements = [ + 'msgpack>=0.5.2', + 'pyzmq>=13.1.0', + 'future', +] + +if sys.version_info < (2, 7): + requirements.append('argparse') + +if sys.version_info < (2, 7): + requirements.append('gevent>=1.1.0,<1.2.0') +elif sys.version_info < (3, 0): + requirements.append('gevent>=1.0') +else: + requirements.append('gevent>=1.1') + +with open("README.rst", "r") as fh: + long_description = fh.read() + setup( name='zerorpc', version=__version__, description='zerorpc is a flexible RPC based on zeromq.', + long_description=long_description, + long_description_content_type='text/x-rst', author=__author__, - url='https://github.com/dotcloud/zerorpc-python', + url='https://github.com/0rpc/zerorpc-python', packages=['zerorpc'], - install_requires=[ - 'argparse', - 'gevent', - 'msgpack-python', - 'pyzmq-static>=2.1.7', - ], - tests_require=['nose'], - test_suite='nose.collector', + install_requires=requirements, + tests_require=['pytest'], + test_suite='pytest.collector', zip_safe=False, - scripts=[ - 'bin/zerorpc' - ], + entry_points={'console_scripts': ['zerorpc = zerorpc.cli:main']}, license='MIT', classifiers=( 'Development Status :: 5 - Production/Stable', @@ -56,5 +75,8 @@ 'Natural Language :: English', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', ), ) diff --git a/tests/test_buffered_channel.py b/tests/test_buffered_channel.py index b223640..20b8173 100644 --- a/tests/test_buffered_channel.py +++ b/tests/test_buffered_channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,41 +23,49 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + +import pytest import gevent +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_bufchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_bufchan.close() - with assert_raises(zerorpc.LostRemote): - client_bufchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() + print('CLIENT LOST SERVER :)') client_bufchan.close() server.close() client.close() @@ -65,31 +73,34 @@ def test_close_server_bufchan(): def test_close_client_bufchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) client_bufchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) server_bufchan.recv() - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_bufchan.close() - with assert_raises(zerorpc.LostRemote): - server_bufchan.recv() - print 'SERVER LOST CLIENT :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() + print('SERVER LOST CLIENT :)') server_bufchan.close() server.close() client.close() @@ -97,29 +108,32 @@ def test_close_client_bufchan(): def test_heartbeat_can_open_channel_server_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_bufchan.close() - with assert_raises(zerorpc.LostRemote): - client_bufchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() + print('CLIENT LOST SERVER :)') client_bufchan.close() server.close() client.close() @@ -127,268 +141,369 @@ def test_heartbeat_can_open_channel_server_close(): def test_heartbeat_can_open_channel_client_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) - - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + def server_fn(): + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan) + try: + while True: + gevent.sleep(1) + finally: + server_bufchan.close() + server_coro = gevent.spawn(server_fn) + + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_bufchan.close() client.close() - with assert_raises(zerorpc.LostRemote): - server_bufchan.recv() - print 'SERVER LOST CLIENT :)' - server_bufchan.close() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_coro.get) + else: + with pytest.raises(zerorpc.LostRemote): + server_coro.get() + print('SERVER LOST CLIENT :)') server.close() def test_do_some_req_rep(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - client_bufchan = zerorpc.BufferedChannel(client_hbchan) - - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) def client_do(): - for x in xrange(20): + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan) + for x in range(20): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): - for x in xrange(20): + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan) + + for x in range(20): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() def test_do_some_req_rep_lost_server(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): - print 'running' + print('running') client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - for x in xrange(10): + for x in range(10): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_bufchan.emit('add', (x, x * x)) - with assert_raises(zerorpc.LostRemote): - event = client_bufchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_bufchan.recv() client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - for x in xrange(10): + for x in range(10): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() def test_do_some_req_rep_lost_client(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - for x in xrange(10): + for x in range(10): client_bufchan.emit('add', (x, x * x)) event = client_bufchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - for x in xrange(10): + for x in range(10): event = server_bufchan.recv() assert event.name == 'add' server_bufchan.emit('OK', (sum(event.args),)) - with assert_raises(zerorpc.LostRemote): - event = server_bufchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_bufchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_bufchan.recv() server_bufchan.close() - server_task = gevent.spawn(server_do) + coro_pool.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.join() client.close() server.close() def test_do_some_req_rep_client_timeout(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_bufchan = zerorpc.BufferedChannel(client_hbchan) - with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): - client_bufchan.emit('sleep', (x,)) - event = client_bufchan.recv(timeout=3) - assert event.name == 'OK' - assert event.args == (x,) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(10): + client_bufchan.emit('sleep', (x,)) + event = client_bufchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(10): + client_bufchan.emit('sleep', (x,)) + event = client_bufchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_bufchan = zerorpc.BufferedChannel(server_hbchan) - with assert_raises(zerorpc.LostRemote): - for x in xrange(20): - event = server_bufchan.recv() - assert event.name == 'sleep' - gevent.sleep(event.args[0]) - server_bufchan.emit('OK', event.args) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(20): + event = server_bufchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_bufchan.emit('OK', event.args) + pytest.raises(zerorpc.LostRemote, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.LostRemote): + for x in range(20): + event = server_bufchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_bufchan.emit('OK', event.args) server_bufchan.close() - server_task = gevent.spawn(server_do) - server_task.get() - client_task.get() + coro_pool.spawn(server_do) + + coro_pool.join() client.close() server.close() -class CongestionError(Exception): - pass - - def test_congestion_control_server_pushing(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - client_bufchan = zerorpc.BufferedChannel(client_hbchan) - - event = server.recv() - server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - server_bufchan = zerorpc.BufferedChannel(server_hbchan) + read_cnt = type('Dummy', (object,), { "value": 0 }) def client_do(): - for x in xrange(200): + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=100) + for x in range(200): event = client_bufchan.recv() assert event.name == 'coucou' assert event.args == x + read_cnt.value += 1 + client_bufchan.close() - client_task = gevent.spawn(client_do) + coro_pool = gevent.pool.Pool() + coro_pool.spawn(client_do) def server_do(): - with assert_raises(CongestionError): - for x in xrange(200): - if server_bufchan.emit('coucou', x, block=False) == False: - raise CongestionError() # will fail when x == 1 + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=100) + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 1 server_bufchan.emit('coucou', 1) # block until receiver is ready - with assert_raises(CongestionError): - for x in xrange(2, 200): - if server_bufchan.emit('coucou', x, block=False) == False: - raise CongestionError() # will fail when x == 100 - for x in xrange(101, 200): + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(2, 200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(2, 200): + server_bufchan.emit('coucou', x, timeout=0) # will fail when x == 100 + for x in range(read_cnt.value, 200): server_bufchan.emit('coucou', x) # block until receiver is ready + server_bufchan.close() + + coro_pool.spawn(server_do) + try: + coro_pool.join() + except zerorpc.LostRemote: + pass + finally: + client.close() + server.close() + + +def test_on_close_if(): + """ + Test that the on_close_if method does not cause exceptions when the client + is slow to recv() data. + """ + endpoint = random_ipc_endpoint() + server_events = zerorpc.Events(zmq.ROUTER) + server_events.bind(endpoint) + server = zerorpc.ChannelMultiplexer(server_events) + + client_events = zerorpc.Events(zmq.DEALER) + client_events.connect(endpoint) + client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) + + client_channel = client.channel() + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=2) + client_bufchan = zerorpc.BufferedChannel(client_hbchan, inqueue_size=10) - server_task = gevent.spawn(server_do) + event = server.recv() + server_channel = server.channel(event) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=2) + server_bufchan = zerorpc.BufferedChannel(server_hbchan, inqueue_size=10) + + seen = [] + + def is_stream_done(event): + return event.name == 'done' + + def client_do(): + while True: + event = client_bufchan.recv() + if event.name == 'done': + return + seen.append(event.args) + gevent.sleep(0.1) + + def server_do(): + for i in range(0, 10): + server_bufchan.emit('blah', (i)) + server_bufchan.emit('done', ('bye')) + + client_bufchan.on_close_if = is_stream_done + + coro_pool = gevent.pool.Pool() + g1 = coro_pool.spawn(client_do) + g2 = coro_pool.spawn(server_do) + + g1.get() # Re-raise any exceptions... + g2.get() + + assert seen == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - server_task.get() - client_task.get() client_bufchan.close() - client.close() - server_task.get() server_bufchan.close() + client.close() server.close() diff --git a/tests/test_channel.py b/tests/test_channel.py index 4eebd65..1d59b1e 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,18 +23,22 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_events_channel_client_side(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -42,24 +46,25 @@ def test_events_channel_client_side(): client_channel.emit('someevent', (42,)) event = server.recv() - print event - assert event.args == (42,) - assert event.header.get('zmqid', None) is not None - - server.emit('someanswer', (21,), - xheader=dict(response_to=event.header['message_id'], - zmqid=event.header['zmqid'])) + print(event) + assert list(event.args) == [42] + assert event.identity is not None + + reply_event = server.new_event('someanswer', (21,), + xheader={'response_to': event.header['message_id']}) + reply_event.identity = event.identity + server.emit_event(reply_event) event = client_channel.recv() - assert event.args == (21,) + assert list(event.args) == [21] def test_events_channel_client_side_server_send_many(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -67,26 +72,27 @@ def test_events_channel_client_side_server_send_many(): client_channel.emit('giveme', (10,)) event = server.recv() - print event - assert event.args == (10,) - assert event.header.get('zmqid', None) is not None - - for x in xrange(10): - server.emit('someanswer', (x,), - xheader=dict(response_to=event.header['message_id'], - zmqid=event.header['zmqid'])) - for x in xrange(10): + print(event) + assert list(event.args) == [10] + assert event.identity is not None + + for x in range(10): + reply_event = server.new_event('someanswer', (x,), + xheader={'response_to': event.header['message_id']}) + reply_event.identity = event.identity + server.emit_event(reply_event) + for x in range(10): event = client_channel.recv() - assert event.args == (x,) + assert list(event.args) == [x] def test_events_channel_both_side(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events) @@ -94,21 +100,21 @@ def test_events_channel_both_side(): client_channel.emit('openthat', (42,)) event = server.recv() - print event - assert event.args == (42,) + print(event) + assert list(event.args) == [42] assert event.name == 'openthat' server_channel = server.channel(event) server_channel.emit('test', (21,)) event = client_channel.recv() - assert event.args == (21,) + assert list(event.args) == [21] assert event.name == 'test' server_channel.emit('test', (22,)) event = client_channel.recv() - assert event.args == (22,) + assert list(event.args) == [22] assert event.name == 'test' server_events.close() diff --git a/tests/test_client.py b/tests/test_client.py index 599b0e1..6a692b3 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,10 +23,11 @@ # SOFTWARE. +from __future__ import absolute_import import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint def test_client_connect(): endpoint = random_ipc_endpoint() diff --git a/tests/test_client_async.py b/tests/test_client_async.py new file mode 100644 index 0000000..ced4b1f --- /dev/null +++ b/tests/test_client_async.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# Open Source Initiative OSI - The MIT License (MIT):Licensing +# +# The MIT License (MIT) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +from __future__ import print_function +from __future__ import absolute_import +import pytest +import gevent +import sys + +from zerorpc import zmq +import zerorpc +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR + + +def test_client_server_client_timeout_with_async(): + endpoint = random_ipc_endpoint() + + class MySrv(zerorpc.Server): + + def lolita(self): + return 42 + + def add(self, a, b): + gevent.sleep(TIME_FACTOR * 10) + return a + b + + srv = MySrv() + srv.bind(endpoint) + gevent.spawn(srv.run) + + client = zerorpc.Client(timeout=TIME_FACTOR * 2) + client.connect(endpoint) + + async_result = client.add(1, 4, async_=True) + + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print(async_result.get()) + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + print(async_result.get()) + client.close() + srv.close() + + +def test_client_server_with_async(): + endpoint = random_ipc_endpoint() + + class MySrv(zerorpc.Server): + + def lolita(self): + return 42 + + def add(self, a, b): + return a + b + + srv = MySrv() + srv.bind(endpoint) + gevent.spawn(srv.run) + + client = zerorpc.Client() + client.connect(endpoint) + + async_result = client.lolita(async_=True) + assert async_result.get() == 42 + + async_result = client.add(1, 4, async_=True) + assert async_result.get() == 5 diff --git a/tests/test_client_heartbeat.py b/tests/test_client_heartbeat.py index e9de869..6b552a4 100644 --- a/tests/test_client_heartbeat.py +++ b/tests/test_client_heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,10 +23,15 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import next +from builtins import range + import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_client_server_hearbeat(): @@ -38,17 +43,17 @@ def lolita(self): return 42 def slow(self): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 1) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=1) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1) client.connect(endpoint) assert client.lolita() == 42 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_server_activate_heartbeat(): @@ -57,19 +62,19 @@ def test_client_server_activate_heartbeat(): class MySrv(zerorpc.Server): def lolita(self): - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) return 42 - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) gevent.sleep(0) - client = zerorpc.Client(heartbeat=1) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4) client.connect(endpoint) assert client.lolita() == 42 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_server_passive_hearbeat(): @@ -81,18 +86,19 @@ def lolita(self): return 42 def slow(self): - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) return 2 - srv = MySrv(heartbeat=1) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) + gevent.sleep(0) - client = zerorpc.Client(heartbeat=1, passive_heartbeat=True) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4, passive_heartbeat=True) client.connect(endpoint) assert client.slow() == 2 - print 'GOT ANSWER' + print('GOT ANSWER') def test_client_hb_doesnt_linger_on_streaming(): @@ -102,18 +108,18 @@ class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): - return xrange(42) + return range(42) - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - assert list(client1.iter()) == list(xrange(42)) - print 'sleep 3s' - gevent.sleep(3) + assert list(client1.iter()) == list(range(42)) + print('sleep 3s') + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -126,18 +132,18 @@ class MySrv(zerorpc.Server): def lolita(self): return 42 - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) - client2 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) - client3 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) + client2 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) + client3 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) assert client1.lolita() == 42 assert client2.lolita() == 42 - gevent.sleep(3) + gevent.sleep(TIME_FACTOR * 3) assert client3.lolita() == 42 @@ -150,18 +156,18 @@ class MySrv(zerorpc.Server): def iter(self): return [] - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - print 'grab iter' + print('grab iter') i = client1.iter() - print 'sleep 3s' - gevent.sleep(3) + print('sleep 3s') + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() @@ -173,22 +179,22 @@ class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): - return xrange(500) + return range(500) - srv = MySrv(heartbeat=1, context=zerorpc.Context()) + srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) - client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) + client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): - print 'grab iter' + print('grab iter') i = client1.iter() - print 'consume some' - assert list(next(i) for x in xrange(142)) == list(xrange(142)) + print('consume some') + assert list(next(i) for x in range(142)) == list(range(142)) - print 'sleep 3s' - gevent.sleep(3) + print('sleep 3s') + gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join() diff --git a/tests/test_events.py b/tests/test_events.py index 3d4766a..7acf98e 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,11 +23,15 @@ # SOFTWARE. +from __future__ import print_function, absolute_import +from builtins import str, bytes +from builtins import range, object + from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint -class MokupContext(): +class MokupContext(object): _next_id = 0 def new_msgid(self): @@ -44,46 +48,46 @@ def test_context(): def test_event(): context = MokupContext() event = zerorpc.Event('mylittleevent', (None,), context=context) - print event + print(event) assert event.name == 'mylittleevent' assert event.header['message_id'] == 0 assert event.args == (None,) event = zerorpc.Event('mylittleevent2', ('42',), context=context) - print event + print(event) assert event.name == 'mylittleevent2' assert event.header['message_id'] == 1 assert event.args == ('42',) event = zerorpc.Event('mylittleevent3', ('a', 42), context=context) - print event + print(event) assert event.name == 'mylittleevent3' assert event.header['message_id'] == 2 assert event.args == ('a', 42) - event = zerorpc.Event('mylittleevent4', ('b', 21), context=context) - print event + event = zerorpc.Event('mylittleevent4', ('', 21), context=context) + print(event) assert event.name == 'mylittleevent4' assert event.header['message_id'] == 3 - assert event.args == ('b', 21) + assert event.args == ('', 21) packed = event.pack() unpacked = zerorpc.Event.unpack(packed) - print unpacked + print(unpacked) assert unpacked.name == 'mylittleevent4' assert unpacked.header['message_id'] == 3 - assert unpacked.args == ('b', 21) + assert list(unpacked.args) == ['', 21] event = zerorpc.Event('mylittleevent5', ('c', 24, True), header={'lol': 'rofl'}, context=None) - print event + print(event) assert event.name == 'mylittleevent5' assert event.header['lol'] == 'rofl' assert event.args == ('c', 24, True) event = zerorpc.Event('mod', (42,), context=context) - print event + print(event) assert event.name == 'mod' assert event.header['message_id'] == 4 assert event.args == (42,) @@ -102,9 +106,9 @@ def test_events_req_rep(): client.emit('myevent', ('arg1',)) event = server.recv() - print event + print(event) assert event.name == 'myevent' - assert event.args == ('arg1',) + assert list(event.args) == ['arg1'] def test_events_req_rep2(): @@ -115,41 +119,42 @@ def test_events_req_rep2(): client = zerorpc.Events(zmq.REQ) client.connect(endpoint) - for i in xrange(10): + for i in range(10): client.emit('myevent' + str(i), (i,)) event = server.recv() - print event + print(event) assert event.name == 'myevent' + str(i) - assert event.args == (i,) + assert list(event.args) == [i] server.emit('answser' + str(i * 2), (i * 2,)) event = client.recv() - print event + print(event) assert event.name == 'answser' + str(i * 2) - assert event.args == (i * 2,) + assert list(event.args) == [i * 2] def test_events_dealer_router(): endpoint = random_ipc_endpoint() - server = zerorpc.Events(zmq.XREP) + server = zerorpc.Events(zmq.ROUTER) server.bind(endpoint) - client = zerorpc.Events(zmq.XREQ) + client = zerorpc.Events(zmq.DEALER) client.connect(endpoint) - for i in xrange(6): + for i in range(6): client.emit('myevent' + str(i), (i,)) event = server.recv() - print event + print(event) assert event.name == 'myevent' + str(i) - assert event.args == (i,) + assert list(event.args) == [i] - server.emit('answser' + str(i * 2), (i * 2,), - xheader=dict(zmqid=event.header['zmqid'])) + reply_event = server.new_event('answser' + str(i * 2), (i * 2,)) + reply_event.identity = event.identity + server.emit_event(reply_event) event = client.recv() - print event + print(event) assert event.name == 'answser' + str(i * 2) - assert event.args == (i * 2,) + assert list(event.args) == [i * 2] def test_events_push_pull(): @@ -160,11 +165,35 @@ def test_events_push_pull(): client = zerorpc.Events(zmq.PUSH) client.connect(endpoint) - for x in xrange(10): + for x in range(10): client.emit('myevent', (x,)) - for x in xrange(10): + for x in range(10): event = server.recv() - print event + print(event) assert event.name == 'myevent' - assert event.args == (x,) + assert list(event.args) == [x] + + +def test_msgpack(): + context = zerorpc.Context() + event = zerorpc.Event(u'myevent', (u'a',), context=context) + print(event) + # note here that str is an unicode string in all Python version (thanks to + # the builtin str import). + assert isinstance(event.name, str) + for key in event.header.keys(): + assert isinstance(key, str) + assert isinstance(event.header[u'message_id'], bytes) + assert isinstance(event.header[u'v'], int) + assert isinstance(event.args[0], str) + + packed = event.pack() + event = event.unpack(packed) + print(event) + assert isinstance(event.name, str) + for key in event.header.keys(): + assert isinstance(key, str) + assert isinstance(event.header[u'message_id'], bytes) + assert isinstance(event.header[u'v'], int) + assert isinstance(event.args[0], str) diff --git a/tests/test_heartbeat.py b/tests/test_heartbeat.py index 13caab5..14c66fd 100644 --- a/tests/test_heartbeat.py +++ b/tests/test_heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,39 +23,47 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + +import pytest import gevent +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_close_server_hbchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_hbchan.recv() - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_hbchan.close() - with assert_raises(zerorpc.LostRemote): - client_hbchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_hbchan.recv() + print('CLIENT LOST SERVER :)') client_hbchan.close() server.close() client.close() @@ -63,29 +71,32 @@ def test_close_server_hbchan(): def test_close_client_hbchan(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) client_hbchan.emit('openthat', None) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) server_hbchan.recv() - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_hbchan.close() - with assert_raises(zerorpc.LostRemote): - server_hbchan.recv() - print 'SERVER LOST CLIENT :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_hbchan.recv() + print('SERVER LOST CLIENT :)') server_hbchan.close() server.close() client.close() @@ -93,27 +104,30 @@ def test_close_client_hbchan(): def test_heartbeat_can_open_channel_server_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - gevent.sleep(3) - print 'CLOSE SERVER SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE SERVER SOCKET!!!') server_hbchan.close() - with assert_raises(zerorpc.LostRemote): - client_hbchan.recv() - print 'CLIENT LOST SERVER :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_hbchan.recv() + print('CLIENT LOST SERVER :)') client_hbchan.close() server.close() client.close() @@ -121,61 +135,64 @@ def test_heartbeat_can_open_channel_server_close(): def test_heartbeat_can_open_channel_client_close(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - gevent.sleep(3) - print 'CLOSE CLIENT SOCKET!!!' + gevent.sleep(TIME_FACTOR * 3) + print('CLOSE CLIENT SOCKET!!!') client_hbchan.close() client.close() - with assert_raises(zerorpc.LostRemote): - server_hbchan.recv() - print 'SERVER LOST CLIENT :)' + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_hbchan.recv() + print('SERVER LOST CLIENT :)') server_hbchan.close() server.close() def test_do_some_req_rep(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 4) event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 4) def client_do(): - for x in xrange(20): + for x in range(20): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) def server_do(): - for x in xrange(20): + for x in range(20): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) @@ -191,26 +208,29 @@ def server_do(): def test_do_some_req_rep_lost_server(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): - print 'running' + print('running') client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - for x in xrange(10): + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + for x in range(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_hbchan.emit('add', (x, x * x)) - with assert_raises(zerorpc.LostRemote): - event = client_hbchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, client_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + client_hbchan.recv() client_hbchan.close() client_task = gevent.spawn(client_do) @@ -218,8 +238,8 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - for x in xrange(10): + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + for x in range(10): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) @@ -235,23 +255,23 @@ def server_do(): def test_do_some_req_rep_lost_client(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): client_hbchan.emit('add', (x, x * x)) event = client_hbchan.recv() assert event.name == 'OK' - assert event.args == (x + x * x,) + assert list(event.args) == [x + x * x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -259,15 +279,18 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) - for x in xrange(10): + for x in range(10): event = server_hbchan.recv() assert event.name == 'add' server_hbchan.emit('OK', (sum(event.args),)) - with assert_raises(zerorpc.LostRemote): - event = server_hbchan.recv() + if sys.version_info < (2, 7): + pytest.raises(zerorpc.LostRemote, server_hbchan.recv) + else: + with pytest.raises(zerorpc.LostRemote): + server_hbchan.recv() server_hbchan.close() server_task = gevent.spawn(server_do) @@ -280,24 +303,33 @@ def server_do(): def test_do_some_req_rep_client_timeout(): endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) + server_events = zerorpc.Events(zmq.ROUTER) server_events.bind(endpoint) server = zerorpc.ChannelMultiplexer(server_events) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) def client_do(): client_channel = client.channel() - client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=1) - - with assert_raises(zerorpc.TimeoutExpired): - for x in xrange(10): - client_hbchan.emit('sleep', (x,)) - event = client_hbchan.recv(timeout=3) - assert event.name == 'OK' - assert event.args == (x,) + client_hbchan = zerorpc.HeartBeatOnChannel(client_channel, freq=TIME_FACTOR * 2) + + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(10): + client_hbchan.emit('sleep', (x,)) + event = client_hbchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] + pytest.raises(zerorpc.TimeoutExpired, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.TimeoutExpired): + for x in range(10): + client_hbchan.emit('sleep', (x,)) + event = client_hbchan.recv(timeout=TIME_FACTOR * 3) + assert event.name == 'OK' + assert list(event.args) == [x] client_hbchan.close() client_task = gevent.spawn(client_do) @@ -305,14 +337,23 @@ def client_do(): def server_do(): event = server.recv() server_channel = server.channel(event) - server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=1) - - with assert_raises(zerorpc.LostRemote): - for x in xrange(20): - event = server_hbchan.recv() - assert event.name == 'sleep' - gevent.sleep(event.args[0]) - server_hbchan.emit('OK', event.args) + server_hbchan = zerorpc.HeartBeatOnChannel(server_channel, freq=TIME_FACTOR * 2) + + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + for x in range(20): + event = server_hbchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_hbchan.emit('OK', event.args) + pytest.raises(zerorpc.LostRemote, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.LostRemote): + for x in range(20): + event = server_hbchan.recv() + assert event.name == 'sleep' + gevent.sleep(TIME_FACTOR * event.args[0]) + server_hbchan.emit('OK', event.args) server_hbchan.close() server_task = gevent.spawn(server_do) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 5545b20..3163a3a 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,15 +23,21 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import str +from future.utils import tobytes + +import pytest import gevent import gevent.local import random -import md5 +import hashlib +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_resolve_endpoint(): @@ -46,16 +52,16 @@ def resolve(endpoint): cnt = c.register_middleware({ 'resolve_endpoint': resolve }) - print 'registered_count:', cnt + print('registered_count:', cnt) assert cnt == 1 - print 'resolve titi:', c.middleware_resolve_endpoint('titi') - assert c.middleware_resolve_endpoint('titi') == test_endpoint + print('resolve titi:', c.hook_resolve_endpoint('titi')) + assert c.hook_resolve_endpoint('titi') == test_endpoint - print 'resolve toto:', c.middleware_resolve_endpoint('toto') - assert c.middleware_resolve_endpoint('toto') == 'toto' + print('resolve toto:', c.hook_resolve_endpoint('toto')) + assert c.hook_resolve_endpoint('toto') == 'toto' - class Resolver(): + class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'toto': @@ -63,26 +69,26 @@ def resolve_endpoint(self, endpoint): return endpoint cnt = c.register_middleware(Resolver()) - print 'registered_count:', cnt + print('registered_count:', cnt) assert cnt == 1 - print 'resolve titi:', c.middleware_resolve_endpoint('titi') - assert c.middleware_resolve_endpoint('titi') == test_endpoint - print 'resolve toto:', c.middleware_resolve_endpoint('toto') - assert c.middleware_resolve_endpoint('toto') == test_endpoint + print('resolve titi:', c.hook_resolve_endpoint('titi')) + assert c.hook_resolve_endpoint('titi') == test_endpoint + print('resolve toto:', c.hook_resolve_endpoint('toto')) + assert c.hook_resolve_endpoint('toto') == test_endpoint c2 = zerorpc.Context() - print 'resolve titi:', c2.middleware_resolve_endpoint('titi') - assert c2.middleware_resolve_endpoint('titi') == 'titi' - print 'resolve toto:', c2.middleware_resolve_endpoint('toto') - assert c2.middleware_resolve_endpoint('toto') == 'toto' + print('resolve titi:', c2.hook_resolve_endpoint('titi')) + assert c2.hook_resolve_endpoint('titi') == 'titi' + print('resolve toto:', c2.hook_resolve_endpoint('toto')) + assert c2.hook_resolve_endpoint('toto') == 'toto' def test_resolve_endpoint_events(): test_endpoint = random_ipc_endpoint() c = zerorpc.Context() - class Resolver(): + class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'some_service': return test_endpoint @@ -90,19 +96,22 @@ def resolve_endpoint(self, endpoint): class Srv(zerorpc.Server): def hello(self): - print 'heee' + print('heee') return 'world' - srv = Srv(heartbeat=1, context=c) - with assert_raises(zmq.ZMQError): - srv.bind('some_service') + srv = Srv(heartbeat=TIME_FACTOR * 1, context=c) + if sys.version_info < (2, 7): + pytest.raises(zmq.ZMQError, srv.bind, 'some_service') + else: + with pytest.raises(zmq.ZMQError): + srv.bind('some_service') cnt = c.register_middleware(Resolver()) assert cnt == 1 srv.bind('some_service') gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=1, context=c) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 1, context=c) client.connect('some_service') assert client.hello() == 'world' @@ -110,152 +119,7 @@ def hello(self): srv.close() -def test_raise_error(): - endpoint = random_ipc_endpoint() - c = zerorpc.Context() - - class DummyRaiser(): - def raise_error(self, event): - pass - - class Srv(zerorpc.Server): - pass - - srv = Srv(context=c) - srv.bind(endpoint) - gevent.spawn(srv.run) - - client = zerorpc.Client(context=c) - client.connect(endpoint) - - with assert_raises(zerorpc.RemoteError): - client.donotexist() - - cnt = c.register_middleware(DummyRaiser()) - assert cnt == 1 - - with assert_raises(zerorpc.RemoteError): - client.donotexist() - - class HorribleEvalRaiser(): - def raise_error(self, event): - (name, msg, tb) = event.args - etype = eval(name) - e = etype(tb) - raise e - - cnt = c.register_middleware(HorribleEvalRaiser()) - assert cnt == 1 - - with assert_raises(NameError): - try: - client.donotexist() - except NameError as e: - print 'got it:', e - raise - - client.close() - srv.close() - - -def test_call_procedure(): - c = zerorpc.Context() - - def test(argument): - return 'ret_real:' + argument - assert c.middleware_call_procedure(test, 'dummy') == 'ret_real:dummy' - - def middleware_1(procedure, *args, **kwargs): - return 'ret_middleware_1:' + procedure(*args, **kwargs) - cnt = c.register_middleware({ - 'call_procedure': middleware_1 - }) - assert cnt == 1 - assert c.middleware_call_procedure(test, 'dummy') == \ - 'ret_middleware_1:ret_real:dummy' - - def middleware_2(procedure, *args, **kwargs): - return 'ret_middleware_2:' + procedure(*args, **kwargs) - cnt = c.register_middleware({ - 'call_procedure': middleware_2 - }) - assert cnt == 1 - assert c.middleware_call_procedure(test, 'dummy') == \ - 'ret_middleware_2:ret_middleware_1:ret_real:dummy' - - def mangle_arguments(procedure, *args, **kwargs): - return procedure(args[0].upper()) - cnt = c.register_middleware({ - 'call_procedure': mangle_arguments - }) - assert cnt == 1 - assert c.middleware_call_procedure(test, 'dummy') == \ - 'ret_middleware_2:ret_middleware_1:ret_real:DUMMY' - - endpoint = random_ipc_endpoint() - - # client/server - class Server(zerorpc.Server): - def test(self, argument): - return 'ret_real:' + argument - server = Server(heartbeat=1, context=c) - server.bind(endpoint) - gevent.spawn(server.run) - client = zerorpc.Client(heartbeat=1, context=c) - client.connect(endpoint) - assert client.test('dummy') == \ - 'ret_middleware_2:ret_middleware_1:ret_real:DUMMY' - client.close() - server.close() - - # push/pull - trigger = gevent.event.Event() - class Puller(zerorpc.Puller): - argument = None - - def test(self, argument): - self.argument = argument - trigger.set() - return self.argument - - puller = Puller(context=c) - puller.bind(endpoint) - gevent.spawn(puller.run) - pusher = zerorpc.Pusher(context=c) - pusher.connect(endpoint) - trigger.clear() - pusher.test('dummy') - trigger.wait() - assert puller.argument == 'DUMMY' - #FIXME: These seems to be broken - # pusher.close() - # puller.close() - - # pub/sub - trigger = gevent.event.Event() - class Subscriber(zerorpc.Subscriber): - argument = None - - def test(self, argument): - self.argument = argument - trigger.set() - return self.argument - - subscriber = Subscriber(context=c) - subscriber.bind(endpoint) - gevent.spawn(subscriber.run) - publisher = zerorpc.Publisher(context=c) - publisher.connect(endpoint) - trigger.clear() - publisher.test('dummy') - trigger.wait() - assert subscriber.argument == 'DUMMY' - #FIXME: These seems to be broken - # publisher.close() - # subscriber.close() - - -class Tracer: +class Tracer(object): '''Used by test_task_context_* tests''' def __init__(self, identity): self._identity = identity @@ -268,19 +132,19 @@ def trace_id(self): def load_task_context(self, event_header): self._locals.trace_id = event_header.get('trace_id', None) - print self._identity, 'load_task_context', self.trace_id + print(self._identity, 'load_task_context', self.trace_id) self._log.append(('load', self.trace_id)) def get_task_context(self): if self.trace_id is None: # just an ugly code to generate a beautiful little hash. - self._locals.trace_id = '<{0}>'.format(md5.md5( - str(random.random())[3:] + self._locals.trace_id = '<{0}>'.format(hashlib.md5( + tobytes(str(random.random())[3:]) ).hexdigest()[0:6].upper()) - print self._identity, 'get_task_context! [make a new one]', self.trace_id + print(self._identity, 'get_task_context! [make a new one]', self.trace_id) self._log.append(('new', self.trace_id)) else: - print self._identity, 'get_task_context! [reuse]', self.trace_id + print(self._identity, 'get_task_context! [reuse]', self.trace_id) self._log.append(('reuse', self.trace_id)) return { 'trace_id': self.trace_id } @@ -295,7 +159,7 @@ def test_task_context(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -342,7 +206,7 @@ def test_task_context_relay(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -353,7 +217,7 @@ def echo(self, msg): c_relay = zerorpc.Client(context=srv_relay_ctx) c_relay.connect(endpoint1) - class SrvRelay: + class SrvRelay(object): def echo(self, msg): return c_relay.echo('relay' + msg) + 'relayed' @@ -398,7 +262,7 @@ def test_task_context_relay_fork(): cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) - class Srv: + class Srv(object): def echo(self, msg): return msg @@ -409,15 +273,15 @@ def echo(self, msg): c_relay = zerorpc.Client(context=srv_relay_ctx) c_relay.connect(endpoint1) - class SrvRelay: + class SrvRelay(object): def echo(self, msg): def dothework(msg): return c_relay.echo(msg) + 'relayed' g = gevent.spawn(zerorpc.fork_task_context(dothework, srv_relay_ctx), 'relay' + msg) - print 'relaying in separate task:', g + print('relaying in separate task:', g) r = g.get() - print 'back to main task' + print('back to main task') return r srv_relay = zerorpc.Server(SrvRelay(), context=srv_relay_ctx) @@ -462,7 +326,7 @@ def test_task_context_pushpull(): trigger = gevent.event.Event() - class Puller: + class Puller(object): def echo(self, msg): trigger.set() @@ -500,7 +364,7 @@ def test_task_context_pubsub(): trigger = gevent.event.Event() - class Subscriber: + class Subscriber(object): def echo(self, msg): trigger.set() @@ -512,38 +376,53 @@ def echo(self, msg): c.connect(endpoint) trigger.clear() - c.echo('pub...') - trigger.wait() + # We need this retry logic to wait that the subscriber.run coroutine starts + # reading (the published messages will go to /dev/null until then). + while not trigger.is_set(): + c.echo('pub...') + if trigger.wait(TIME_FACTOR * 1): + break subscriber.stop() subscriber_task.join() - assert publisher_tracer._log == [ - ('new', publisher_tracer.trace_id), - ] - assert subscriber_tracer._log == [ - ('load', publisher_tracer.trace_id), - ] + print(publisher_tracer._log) + assert ('new', publisher_tracer.trace_id) in publisher_tracer._log + print(subscriber_tracer._log) + assert ('load', publisher_tracer.trace_id) in subscriber_tracer._log + + +class InspectExceptionMiddleware(Tracer): + def __init__(self, barrier=None): + self.called = False + self._barrier = barrier + Tracer.__init__(self, identity='[server]') + + def server_inspect_exception(self, request_event, reply_event, task_context, exc_info): + assert 'trace_id' in task_context + assert request_event.name == 'echo' + if self._barrier: # Push/Pull + assert reply_event is None + else: # Req/Rep or Req/Stream + assert reply_event.name == 'ERR' + exc_type, exc_value, exc_traceback = exc_info + self.called = True + if self._barrier: + self._barrier.set() -def test_inspect_error_middleware(): +class Srv(object): - class InspectErrorMiddleware(Tracer): - def __init__(self): - self.called = False - Tracer.__init__(self, identity='[server]') + def echo(self, msg): + raise RuntimeError(msg) - def inspect_error(self, task_context, exc_info): - assert 'trace_id' in task_context - exc_type, exc_value, exc_traceback = exc_info - self.called = True - - class Srv(object): - def echo(self, msg): - raise RuntimeError(msg) + @zerorpc.stream + def echoes(self, msg): + raise RuntimeError(msg) +def test_server_inspect_exception_middleware(): endpoint = random_ipc_endpoint() - middleware = InspectErrorMiddleware() + middleware = InspectExceptionMiddleware() ctx = zerorpc.Context() ctx.register_middleware(middleware) @@ -556,7 +435,7 @@ def echo(self, msg): client.connect(endpoint) try: - client.echo('This is a test which should call the InspectErrorMiddleware') + client.echo('This is a test which should call the InspectExceptionMiddleware') except zerorpc.exceptions.RemoteError as ex: assert ex.name == 'RuntimeError' @@ -565,28 +444,11 @@ def echo(self, msg): assert middleware.called is True -def test_inspect_error_middleware_puller(): - - class InspectErrorMiddleware(Tracer): - def __init__(self, barrier): - self.called = False - self._barrier = barrier - Tracer.__init__(self, identity='[server]') - - def inspect_error(self, task_context, exc_info): - assert 'trace_id' in task_context - exc_type, exc_value, exc_traceback = exc_info - self.called = True - self._barrier.set() - - class Srv(object): - def echo(self, msg): - raise RuntimeError(msg) - +def test_server_inspect_exception_middleware_puller(): endpoint = random_ipc_endpoint() barrier = gevent.event.Event() - middleware = InspectErrorMiddleware(barrier) + middleware = InspectExceptionMiddleware(barrier) ctx = zerorpc.Context() ctx.register_middleware(middleware) @@ -599,8 +461,33 @@ def echo(self, msg): client.connect(endpoint) barrier.clear() - client.echo('This is a test which should call the InspectErrorMiddleware') - barrier.wait() + client.echo('This is a test which should call the InspectExceptionMiddleware') + barrier.wait(timeout=TIME_FACTOR * 2) + + client.close() + server.close() + + assert middleware.called is True + +def test_server_inspect_exception_middleware_stream(): + endpoint = random_ipc_endpoint() + + middleware = InspectExceptionMiddleware() + ctx = zerorpc.Context() + ctx.register_middleware(middleware) + + module = Srv() + server = zerorpc.Server(module, context=ctx) + server.bind(endpoint) + gevent.spawn(server.run) + + client = zerorpc.Client() + client.connect(endpoint) + + try: + client.echo('This is a test which should call the InspectExceptionMiddleware') + except zerorpc.exceptions.RemoteError as ex: + assert ex.name == 'RuntimeError' client.close() server.close() diff --git a/tests/test_middleware_before_after_exec.py b/tests/test_middleware_before_after_exec.py new file mode 100644 index 0000000..5dafeb0 --- /dev/null +++ b/tests/test_middleware_before_after_exec.py @@ -0,0 +1,323 @@ +# -*- coding: utf-8 -*- +# Open Source Initiative OSI - The MIT License (MIT):Licensing +# +# The MIT License (MIT) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import +from builtins import range + +import gevent +import zerorpc + +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR + +class EchoModule(object): + + def __init__(self, trigger=None): + self.last_msg = None + self._trigger = trigger + + def echo(self, msg): + self.last_msg = 'echo: ' + msg + if self._trigger: + self._trigger.set() + return self.last_msg + + @zerorpc.stream + def echoes(self, msg): + self.last_msg = 'echo: ' + msg + for i in range(0, 3): + yield self.last_msg + +class ServerBeforeExecMiddleware(object): + + def __init__(self): + self.called = False + + def server_before_exec(self, request_event): + assert request_event.name == "echo" or request_event.name == "echoes" + self.called = True + +def test_hook_server_before_exec(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Client() + test_client.connect(endpoint) + + # Test without a middleware + assert test_client.echo("test") == "echo: test" + + # Test with a middleware + test_middleware = ServerBeforeExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + assert test_client.echo("test") == "echo: test" + assert test_middleware.called == True + + test_server.stop() + test_server_task.join() + +def test_hook_server_before_exec_puller(): + zero_ctx = zerorpc.Context() + trigger = gevent.event.Event() + endpoint = random_ipc_endpoint() + + echo_module = EchoModule(trigger) + test_server = zerorpc.Puller(echo_module, context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Pusher() + test_client.connect(endpoint) + + # Test without a middleware + test_client.echo("test") + trigger.wait(timeout=TIME_FACTOR * 2) + assert echo_module.last_msg == "echo: test" + trigger.clear() + + # Test with a middleware + test_middleware = ServerBeforeExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + test_client.echo("test with a middleware") + trigger.wait(timeout=TIME_FACTOR * 2) + assert echo_module.last_msg == "echo: test with a middleware" + assert test_middleware.called == True + + test_server.stop() + test_server_task.join() + +def test_hook_server_before_exec_stream(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Client() + test_client.connect(endpoint) + + # Test without a middleware + for echo in test_client.echoes("test"): + assert echo == "echo: test" + + # Test with a middleware + test_middleware = ServerBeforeExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + it = test_client.echoes("test") + assert test_middleware.called == True + assert next(it) == "echo: test" + for echo in it: + assert echo == "echo: test" + + test_server.stop() + test_server_task.join() + +class ServerAfterExecMiddleware(object): + + def __init__(self): + self.called = False + + def server_after_exec(self, request_event, reply_event): + self.called = True + self.request_event_name = getattr(request_event, 'name', None) + self.reply_event_name = getattr(reply_event, 'name', None) + +def test_hook_server_after_exec(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Client() + test_client.connect(endpoint) + + # Test without a middleware + assert test_client.echo("test") == "echo: test" + + # Test with a middleware + test_middleware = ServerAfterExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + assert test_client.echo("test") == "echo: test" + assert test_middleware.called == True + assert test_middleware.request_event_name == 'echo' + assert test_middleware.reply_event_name == 'OK' + + test_server.stop() + test_server_task.join() + +def test_hook_server_after_exec_puller(): + zero_ctx = zerorpc.Context() + trigger = gevent.event.Event() + endpoint = random_ipc_endpoint() + + echo_module = EchoModule(trigger) + test_server = zerorpc.Puller(echo_module, context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Pusher() + test_client.connect(endpoint) + + # Test without a middleware + test_client.echo("test") + trigger.wait(timeout=TIME_FACTOR * 2) + assert echo_module.last_msg == "echo: test" + trigger.clear() + + # Test with a middleware + test_middleware = ServerAfterExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + test_client.echo("test with a middleware") + trigger.wait(timeout=TIME_FACTOR * 2) + assert echo_module.last_msg == "echo: test with a middleware" + assert test_middleware.called == True + assert test_middleware.request_event_name == 'echo' + assert test_middleware.reply_event_name is None + + test_server.stop() + test_server_task.join() + +def test_hook_server_after_exec_stream(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Client() + test_client.connect(endpoint) + + # Test without a middleware + for echo in test_client.echoes("test"): + assert echo == "echo: test" + + # Test with a middleware + test_middleware = ServerAfterExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + it = test_client.echoes("test") + assert next(it) == "echo: test" + assert test_middleware.called == False + for echo in it: + assert echo == "echo: test" + assert test_middleware.called == True + assert test_middleware.request_event_name == 'echoes' + assert test_middleware.reply_event_name == 'STREAM_DONE' + + test_server.stop() + test_server_task.join() + +class BrokenEchoModule(object): + + def __init__(self, trigger=None): + self.last_msg = None + self._trigger = trigger + + def echo(self, msg): + try: + self.last_msg = "Raise" + raise RuntimeError("BrokenEchoModule") + finally: + if self._trigger: + self._trigger.set() + + @zerorpc.stream + def echoes(self, msg): + self.echo(msg) + +def test_hook_server_after_exec_on_error(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(BrokenEchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Client() + test_client.connect(endpoint) + + test_middleware = ServerAfterExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + try: + test_client.echo("test") + except zerorpc.RemoteError: + pass + assert test_middleware.called == False + + test_server.stop() + test_server_task.join() + +def test_hook_server_after_exec_on_error_puller(): + zero_ctx = zerorpc.Context() + trigger = gevent.event.Event() + endpoint = random_ipc_endpoint() + + echo_module = BrokenEchoModule(trigger) + test_server = zerorpc.Puller(echo_module, context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Pusher() + test_client.connect(endpoint) + + test_middleware = ServerAfterExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + try: + test_client.echo("test with a middleware") + trigger.wait(timeout=TIME_FACTOR * 2) + except zerorpc.RemoteError: + pass + assert echo_module.last_msg == "Raise" + assert test_middleware.called == False + + test_server.stop() + test_server_task.join() + +def test_hook_server_after_exec_on_error_stream(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(BrokenEchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + test_client = zerorpc.Client() + test_client.connect(endpoint) + + test_middleware = ServerAfterExecMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + try: + test_client.echoes("test") + except zerorpc.RemoteError: + pass + assert test_middleware.called == False + + test_server.stop() + test_server_task.join() diff --git a/tests/test_middleware_client.py b/tests/test_middleware_client.py new file mode 100644 index 0000000..943985e --- /dev/null +++ b/tests/test_middleware_client.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# Open Source Initiative OSI - The MIT License (MIT):Licensing +# +# The MIT License (MIT) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import absolute_import +from builtins import range + +import gevent +import zerorpc + +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR + +class EchoModule(object): + + def __init__(self, trigger=None): + self.last_msg = None + self._trigger = trigger + + def echo(self, msg): + self.last_msg = "echo: " + msg + if self._trigger: + self._trigger.set() + return self.last_msg + + @zerorpc.stream + def echoes(self, msg): + self.last_msg = "echo: " + msg + for i in range(0, 3): + yield self.last_msg + + def crash(self, msg): + try: + self.last_msg = "raise: " + msg + raise RuntimeError("BrokenEchoModule") + finally: + if self._trigger: + self._trigger.set() + + @zerorpc.stream + def echoes_crash(self, msg): + self.crash(msg) + + def timeout(self, msg): + self.last_msg = "timeout: " + msg + gevent.sleep(TIME_FACTOR * 2) + +def test_hook_client_before_request(): + + class ClientBeforeRequestMiddleware(object): + def __init__(self): + self.called = False + def client_before_request(self, event): + self.called = True + self.method = event.name + + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(context=zero_ctx) + test_client.connect(endpoint) + + assert test_client.echo("test") == "echo: test" + + test_middleware = ClientBeforeRequestMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + assert test_client.echo("test") == "echo: test" + assert test_middleware.called == True + assert test_middleware.method == 'echo' + + test_server.stop() + test_server_task.join() + +class ClientAfterRequestMiddleware(object): + def __init__(self): + self.called = False + def client_after_request(self, req_event, rep_event, exception): + self.called = True + assert req_event is not None + assert req_event.name == "echo" or req_event.name == "echoes" + self.retcode = rep_event.name + assert exception is None + +def test_hook_client_after_request(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(context=zero_ctx) + test_client.connect(endpoint) + + assert test_client.echo("test") == "echo: test" + + test_middleware = ClientAfterRequestMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + assert test_client.echo("test") == "echo: test" + assert test_middleware.called == True + assert test_middleware.retcode == 'OK' + + test_server.stop() + test_server_task.join() + +def test_hook_client_after_request_stream(): + zero_ctx = zerorpc.Context() + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(context=zero_ctx) + test_client.connect(endpoint) + + it = test_client.echoes("test") + assert next(it) == "echo: test" + for echo in it: + assert echo == "echo: test" + + test_middleware = ClientAfterRequestMiddleware() + zero_ctx.register_middleware(test_middleware) + assert test_middleware.called == False + it = test_client.echoes("test") + assert next(it) == "echo: test" + assert test_middleware.called == False + for echo in it: + assert echo == "echo: test" + assert test_middleware.called == True + assert test_middleware.retcode == 'STREAM_DONE' + + test_server.stop() + test_server_task.join() + +def test_hook_client_after_request_timeout(): + + class ClientAfterRequestMiddleware(object): + def __init__(self): + self.called = False + def client_after_request(self, req_event, rep_event, exception): + self.called = True + assert req_event is not None + assert req_event.name == "timeout" + assert rep_event is None + + zero_ctx = zerorpc.Context() + test_middleware = ClientAfterRequestMiddleware() + zero_ctx.register_middleware(test_middleware) + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) + test_client.connect(endpoint) + + assert test_middleware.called == False + try: + test_client.timeout("test") + except zerorpc.TimeoutExpired as ex: + assert test_middleware.called == True + assert "timeout" in ex.args[0] + + test_server.stop() + test_server_task.join() + +class ClientAfterFailedRequestMiddleware(object): + def __init__(self): + self.called = False + def client_after_request(self, req_event, rep_event, exception): + assert req_event is not None + assert req_event.name == "crash" or req_event.name == "echoes_crash" + self.called = True + assert isinstance(exception, zerorpc.RemoteError) + assert exception.name == 'RuntimeError' + assert 'BrokenEchoModule' in exception.msg + assert rep_event.name == 'ERR' + +def test_hook_client_after_request_remote_error(): + + zero_ctx = zerorpc.Context() + test_middleware = ClientAfterFailedRequestMiddleware() + zero_ctx.register_middleware(test_middleware) + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) + test_client.connect(endpoint) + + assert test_middleware.called == False + try: + test_client.crash("test") + except zerorpc.RemoteError: + assert test_middleware.called == True + + test_server.stop() + test_server_task.join() + +def test_hook_client_after_request_remote_error_stream(): + + zero_ctx = zerorpc.Context() + test_middleware = ClientAfterFailedRequestMiddleware() + zero_ctx.register_middleware(test_middleware) + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) + test_client.connect(endpoint) + + assert test_middleware.called == False + try: + test_client.echoes_crash("test") + except zerorpc.RemoteError: + assert test_middleware.called == True + + test_server.stop() + test_server_task.join() + +def test_hook_client_handle_remote_error_inspect(): + + class ClientHandleRemoteErrorMiddleware(object): + def __init__(self): + self.called = False + def client_handle_remote_error(self, event): + self.called = True + + test_middleware = ClientHandleRemoteErrorMiddleware() + zero_ctx = zerorpc.Context() + zero_ctx.register_middleware(test_middleware) + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(context=zero_ctx) + test_client.connect(endpoint) + + assert test_middleware.called == False + try: + test_client.crash("test") + except zerorpc.RemoteError as ex: + assert test_middleware.called == True + assert ex.name == "RuntimeError" + + test_server.stop() + test_server_task.join() + +# This is a seriously broken idea, but possible nonetheless +class ClientEvalRemoteErrorMiddleware(object): + def __init__(self): + self.called = False + def client_handle_remote_error(self, event): + self.called = True + name, msg, tb = event.args + etype = eval(name) + e = etype(tb) + return e + +def test_hook_client_handle_remote_error_eval(): + test_middleware = ClientEvalRemoteErrorMiddleware() + zero_ctx = zerorpc.Context() + zero_ctx.register_middleware(test_middleware) + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(context=zero_ctx) + test_client.connect(endpoint) + + assert test_middleware.called == False + try: + test_client.crash("test") + except RuntimeError as ex: + assert test_middleware.called == True + assert "BrokenEchoModule" in ex.args[0] + + test_server.stop() + test_server_task.join() + +def test_hook_client_handle_remote_error_eval_stream(): + test_middleware = ClientEvalRemoteErrorMiddleware() + zero_ctx = zerorpc.Context() + zero_ctx.register_middleware(test_middleware) + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(context=zero_ctx) + test_client.connect(endpoint) + + assert test_middleware.called == False + try: + test_client.echoes_crash("test") + except RuntimeError as ex: + assert test_middleware.called == True + assert "BrokenEchoModule" in ex.args[0] + + test_server.stop() + test_server_task.join() + +def test_hook_client_after_request_custom_error(): + + # This is a seriously broken idea, but possible nonetheless + class ClientEvalInspectRemoteErrorMiddleware(object): + def __init__(self): + self.called = False + def client_handle_remote_error(self, event): + name, msg, tb = event.args + etype = eval(name) + e = etype(tb) + return e + def client_after_request(self, req_event, rep_event, exception): + assert req_event is not None + assert req_event.name == "crash" + self.called = True + assert isinstance(exception, RuntimeError) + + test_middleware = ClientEvalInspectRemoteErrorMiddleware() + zero_ctx = zerorpc.Context() + zero_ctx.register_middleware(test_middleware) + endpoint = random_ipc_endpoint() + + test_server = zerorpc.Server(EchoModule(), context=zero_ctx) + test_server.bind(endpoint) + test_server_task = gevent.spawn(test_server.run) + + test_client = zerorpc.Client(context=zero_ctx) + test_client.connect(endpoint) + + assert test_middleware.called == False + try: + test_client.crash("test") + except RuntimeError as ex: + assert test_middleware.called == True + assert "BrokenEchoModule" in ex.args[0] + + test_server.stop() + test_server_task.join() diff --git a/tests/test_pubpush.py b/tests/test_pubpush.py index 2db0787..a99f9b4 100644 --- a/tests/test_pubpush.py +++ b/tests/test_pubpush.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,11 +23,15 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + import gevent import gevent.event - import zerorpc -from testutils import teardown, random_ipc_endpoint + +from .testutils import teardown, random_ipc_endpoint def test_pushpull_inheritance(): @@ -39,7 +43,7 @@ def test_pushpull_inheritance(): class Puller(zerorpc.Puller): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -50,7 +54,7 @@ def lolita(self, a, b): trigger.clear() pusher.lolita(1, 2) trigger.wait() - print 'done' + print('done') def test_pubsub_inheritance(): @@ -62,7 +66,7 @@ def test_pubsub_inheritance(): class Subscriber(zerorpc.Subscriber): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -71,10 +75,15 @@ def lolita(self, a, b): gevent.spawn(subscriber.run) trigger.clear() - publisher.lolita(1, 2) - trigger.wait() - print 'done' + # We need this retry logic to wait that the subscriber.run coroutine starts + # reading (the published messages will go to /dev/null until then). + for attempt in range(0, 10): + publisher.lolita(1, 2) + if trigger.wait(0.2): + print('done') + return + raise RuntimeError("The subscriber didn't receive any published message") def test_pushpull_composite(): endpoint = random_ipc_endpoint() @@ -82,7 +91,7 @@ def test_pushpull_composite(): class Puller(object): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -97,7 +106,7 @@ def lolita(self, a, b): trigger.clear() pusher.lolita(1, 2) trigger.wait() - print 'done' + print('done') def test_pubsub_composite(): @@ -106,7 +115,7 @@ def test_pubsub_composite(): class Subscriber(object): def lolita(self, a, b): - print 'lolita', a, b + print('lolita', a, b) assert a + b == 3 trigger.set() @@ -119,6 +128,12 @@ def lolita(self, a, b): gevent.spawn(subscriber.run) trigger.clear() - publisher.lolita(1, 2) - trigger.wait() - print 'done' + # We need this retry logic to wait that the subscriber.run coroutine starts + # reading (the published messages will go to /dev/null until then). + for attempt in range(0, 10): + publisher.lolita(1, 2) + if trigger.wait(0.2): + print('done') + return + + raise RuntimeError("The subscriber didn't receive any published message") diff --git a/tests/test_reqstream.py b/tests/test_reqstream.py index 6b79ddb..71e1511 100644 --- a/tests/test_reqstream.py +++ b/tests/test_reqstream.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,10 +23,21 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + import gevent import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR + +try: + # Try collections.abc for 3.4+ + from collections.abc import Iterator +except ImportError: + # Fallback to collections for Python 2 + from collections import Iterator def test_rcp_streaming(): @@ -36,27 +47,27 @@ class MySrv(zerorpc.Server): @zerorpc.rep def range(self, max): - return range(max) + return list(range(max)) @zerorpc.stream def xrange(self, max): - return xrange(max) + return range(max) - srv = MySrv(heartbeat=2) + srv = MySrv(heartbeat=TIME_FACTOR * 4) srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(heartbeat=2) + client = zerorpc.Client(heartbeat=TIME_FACTOR * 4) client.connect(endpoint) r = client.range(10) - assert r == tuple(range(10)) + assert list(r) == list(range(10)) r = client.xrange(10) - assert getattr(r, 'next', None) is not None + assert isinstance(r, Iterator) l = [] - print 'wait 4s for fun' - gevent.sleep(4) + print('wait 4s for fun') + gevent.sleep(TIME_FACTOR * 4) for x in r: l.append(x) - assert l == range(10) + assert l == list(range(10)) diff --git a/tests/test_server.py b/tests/test_server.py index 0dc5ce7..86997a9 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,12 +23,17 @@ # SOFTWARE. -from nose.tools import assert_raises +from __future__ import print_function +from __future__ import absolute_import +from builtins import range + +import pytest import gevent +import sys from zerorpc import zmq import zerorpc -from testutils import teardown, random_ipc_endpoint +from .testutils import teardown, random_ipc_endpoint, TIME_FACTOR def test_server_manual(): @@ -46,20 +51,20 @@ def add(self, a, b): srv.bind(endpoint) gevent.spawn(srv.run) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) client_channel = client.channel() client_channel.emit('lolita', tuple()) event = client_channel.recv() - assert event.args == (42,) + assert list(event.args) == [42] client_channel.close() client_channel = client.channel() client_channel.emit('add', (1, 2)) event = client_channel.recv() - assert event.args == (3,) + assert list(event.args) == [3] client_channel.close() srv.stop() @@ -82,10 +87,10 @@ def add(self, a, b): client = zerorpc.Client() client.connect(endpoint) - print client.lolita() + print(client.lolita()) assert client.lolita() == 42 - print client.add(1, 4) + print(client.add(1, 4)) assert client.add(1, 4) == 5 @@ -98,18 +103,21 @@ def lolita(self): return 42 def add(self, a, b): - gevent.sleep(10) + gevent.sleep(TIME_FACTOR * 10) return a + b srv = MySrv() srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - with assert_raises(zerorpc.TimeoutExpired): - print client.add(1, 4) + if sys.version_info < (2, 7): + pytest.raises(zerorpc.TimeoutExpired, client.add, 1, 4) + else: + with pytest.raises(zerorpc.TimeoutExpired): + print(client.add(1, 4)) client.close() srv.close() @@ -126,12 +134,17 @@ def raise_something(self, a): srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - with assert_raises(zerorpc.RemoteError): - print client.raise_something(42) - assert client.raise_something(range(5)) == 4 + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print(client.raise_something(42)) + pytest.raises(zerorpc.RemoteError, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.RemoteError): + print(client.raise_something(42)) + assert client.raise_something(list(range(5))) == 4 client.close() srv.close() @@ -148,17 +161,22 @@ def raise_error(self): srv.bind(endpoint) gevent.spawn(srv.run) - client = zerorpc.Client(timeout=2) + client = zerorpc.Client(timeout=TIME_FACTOR * 2) client.connect(endpoint) - with assert_raises(zerorpc.RemoteError): - print client.raise_error() + if sys.version_info < (2, 7): + def _do_with_assert_raises(): + print(client.raise_error()) + pytest.raises(zerorpc.RemoteError, _do_with_assert_raises) + else: + with pytest.raises(zerorpc.RemoteError): + print(client.raise_error()) try: client.raise_error() except zerorpc.RemoteError as e: - print 'got that:', e - print 'name', e.name - print 'msg', e.msg + print('got that:', e) + print('name', e.name) + print('msg', e.msg) assert e.name == 'RuntimeError' assert e.msg == 'oops!' @@ -176,28 +194,38 @@ class MySrv(zerorpc.Server): srv.bind(endpoint) gevent.spawn(srv.run) - client_events = zerorpc.Events(zmq.XREQ) + client_events = zerorpc.Events(zmq.DEALER) client_events.connect(endpoint) client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) rpccall = client.channel() rpccall.emit('donotexist', tuple()) event = rpccall.recv() - print event + print(event) assert event.name == 'ERR' (name, msg, tb) = event.args - print 'detailed error', name, msg, tb + print('detailed error', name, msg, tb) assert name == 'NameError' assert msg == 'donotexist' rpccall = client.channel() - rpccall.emit('donotexist', tuple(), xheader=dict(v=1)) + rpccall.emit('donotexist', tuple(), xheader={'v': 1}) event = rpccall.recv() - print event + print(event) assert event.name == 'ERR' (msg,) = event.args - print 'msg only', msg + print('msg only', msg) assert msg == "NameError('donotexist',)" client_events.close() srv.close() + + +def test_removed_unscriptable_error_format_args_spec(): + + class MySrv(zerorpc.Server): + pass + + srv = MySrv() + return_value = srv._format_args_spec(None) + assert return_value is None diff --git a/tests/test_wrapped_events.py b/tests/test_wrapped_events.py deleted file mode 100644 index c631f6d..0000000 --- a/tests/test_wrapped_events.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- coding: utf-8 -*- -# Open Source Initiative OSI - The MIT License (MIT):Licensing -# -# The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import random - -from zerorpc import zmq -import zerorpc -from testutils import teardown, random_ipc_endpoint - - -def test_sub_events(): - endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) - server_events.bind(endpoint) - server = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.XREQ) - client_events.connect(endpoint) - client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - - client_channel = client.channel() - client_channel_events = zerorpc.WrappedEvents(client_channel) - client_channel_events.emit('coucou', 42) - - event = server.recv() - print event - assert type(event.args) is tuple - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - server_channel_channel = zerorpc.ChannelMultiplexer(server_channel_events) - event = server_channel_channel.recv() - print event - assert event.name == 'coucou' - assert event.args == 42 - - server_events.close() - client_events.close() - - -def test_multiple_sub_events(): - endpoint = random_ipc_endpoint() - server_events = zerorpc.Events(zmq.XREP) - server_events.bind(endpoint) - server = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.XREQ) - client_events.connect(endpoint) - client = zerorpc.ChannelMultiplexer(client_events, ignore_broadcast=True) - - client_channel1 = client.channel() - client_channel_events1 = zerorpc.WrappedEvents(client_channel1) - client_channel2 = client.channel() - client_channel_events2 = zerorpc.WrappedEvents(client_channel2) - client_channel_events1.emit('coucou1', 43) - client_channel_events2.emit('coucou2', 44) - client_channel_events2.emit('another', 42) - - event = server.recv() - print event - assert type(event.args) is tuple - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - event = server_channel_events.recv() - print event - assert event.name == 'coucou1' - assert event.args == 43 - - event = server.recv() - print event - assert type(event.args) is tuple - assert event.name == 'w' - subevent = event.args - print 'subevent:', subevent - server_channel = server.channel(event) - server_channel_events = zerorpc.WrappedEvents(server_channel) - event = server_channel_events.recv() - print event - assert event.name == 'coucou2' - assert event.args == 44 - - event = server_channel_events.recv() - print event - assert event.name == 'another' - assert event.args == 42 - - server_events.close() - client_events.close() - - -def test_recursive_multiplexer(): - endpoint = random_ipc_endpoint() - - server_events = zerorpc.Events(zmq.XREP) - server_events.bind(endpoint) - servermux = zerorpc.ChannelMultiplexer(server_events) - - client_events = zerorpc.Events(zmq.XREQ) - client_events.connect(endpoint) - clientmux = zerorpc.ChannelMultiplexer(client_events, - ignore_broadcast=True) - - def ping_pong(climux, srvmux): - cli_chan = climux.channel() - someid = random.randint(0, 1000000) - print 'ping...' - cli_chan.emit('ping', someid) - print 'srv_chan got:' - event = srvmux.recv() - srv_chan = srvmux.channel(event) - print event - assert event.name == 'ping' - assert event.args == someid - print 'pong...' - srv_chan.emit('pong', someid) - print 'cli_chan got:' - event = cli_chan.recv() - print event - assert event.name == 'pong' - assert event.args == someid - srv_chan.close() - cli_chan.close() - - def create_sub_multiplexer(events, from_event=None, - ignore_broadcast=False): - channel = events.channel(from_event) - sub_events = zerorpc.WrappedEvents(channel) - sub_multiplexer = zerorpc.ChannelMultiplexer(sub_events, - ignore_broadcast=ignore_broadcast) - return sub_multiplexer - - def open_sub_multiplexer(climux, srvmux): - someid = random.randint(0, 1000000) - print 'open...' - clisubmux = create_sub_multiplexer(climux, ignore_broadcast=True) - clisubmux.emit('open that', someid) - print 'srvsubmux got:' - event = srvmux.recv() - assert event.name == 'w' - srvsubmux = create_sub_multiplexer(srvmux, event) - event = srvsubmux.recv() - print event - return (clisubmux, srvsubmux) - - ping_pong(clientmux, servermux) - - (clientmux_lv2, servermux_lv2) = open_sub_multiplexer(clientmux, servermux) - ping_pong(clientmux_lv2, servermux_lv2) - - (clientmux_lv3, servermux_lv3) = open_sub_multiplexer(clientmux_lv2, - servermux_lv2) - ping_pong(clientmux_lv3, servermux_lv3) - - (clientmux_lv4, servermux_lv4) = open_sub_multiplexer(clientmux_lv3, - servermux_lv3) - ping_pong(clientmux_lv4, servermux_lv4) - - ping_pong(clientmux_lv4, servermux_lv4) - ping_pong(clientmux_lv3, servermux_lv3) - ping_pong(clientmux_lv2, servermux_lv2) - ping_pong(clientmux, servermux) - ping_pong(clientmux, servermux) - ping_pong(clientmux_lv2, servermux_lv2) - ping_pong(clientmux_lv4, servermux_lv4) - ping_pong(clientmux_lv3, servermux_lv3) - - (clientmux_lv5, servermux_lv5) = open_sub_multiplexer(clientmux_lv4, - servermux_lv4) - ping_pong(clientmux_lv5, servermux_lv5) diff --git a/tests/test_zmq.py b/tests/test_zmq.py index ca71210..1e7b4dd 100644 --- a/tests/test_zmq.py +++ b/tests/test_zmq.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,25 +23,26 @@ # SOFTWARE. +from __future__ import print_function +from __future__ import absolute_import import gevent -import gevent.local -import gevent.queue -import gevent.event from zerorpc import zmq +from .testutils import teardown, random_ipc_endpoint def test1(): + endpoint = random_ipc_endpoint() def server(): c = zmq.Context() s = c.socket(zmq.REP) - s.bind('tcp://0.0.0.0:9999') + s.bind(endpoint) while True: - print 'srv recving...' + print(b'srv recving...') r = s.recv() - print 'srv', r - print 'srv sending...' - s.send('world') + print('srv', r) + print('srv sending...') + s.send(b'world') s.close() c.term() @@ -49,13 +50,13 @@ def server(): def client(): c = zmq.Context() s = c.socket(zmq.REQ) - s.connect('tcp://localhost:9999') + s.connect(endpoint) - print 'cli sending...' - s.send('hello') - print 'cli recving...' + print('cli sending...') + s.send(b'hello') + print('cli recving...') r = s.recv() - print 'cli', r + print('cli', r) s.close() c.term() diff --git a/tests/testutils.py b/tests/testutils.py index 14b084f..85b6a96 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,6 +22,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import print_function +from builtins import str + +import functools +import pytest import random import os @@ -36,9 +41,23 @@ def random_ipc_endpoint(): def teardown(): global _tmpfiles for tmpfile in _tmpfiles: - print 'unlink', tmpfile + print('unlink', tmpfile) try: os.unlink(tmpfile) except Exception: pass _tmpfiles = [] + +def skip(reason): + def _skip(test): + @functools.wraps(test) + def wrap(): + raise pytest.SkipTest(reason) + return wrap + return _skip + +try: + TIME_FACTOR = float(os.environ.get('ZPC_TEST_TIME_FACTOR')) +except TypeError: + TIME_FACTOR = 0.2 +print('ZPC_TEST_TIME_FACTOR:', TIME_FACTOR) diff --git a/tests/zmqbug.py b/tests/zmqbug.py index 2d385f0..1d102a2 100644 --- a/tests/zmqbug.py +++ b/tests/zmqbug.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -25,6 +25,8 @@ # Based on https://github.com/traviscline/gevent-zeromq/blob/master/gevent_zeromq/core.py +from __future__ import print_function + import zmq import gevent.event @@ -79,7 +81,7 @@ def send(self, data, flags=0, copy=True, track=False): while True: try: return super(ZMQSocket, self).send(data, flags, copy, track) - except zmq.ZMQError, e: + except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: raise self._writable.clear() @@ -92,14 +94,14 @@ def recv(self, flags=0, copy=True, track=False): while True: try: return super(ZMQSocket, self).recv(flags, copy, track) - except zmq.ZMQError, e: + except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: raise self._readable.clear() while not self._readable.wait(timeout=10): events = self.getsockopt(zmq.EVENTS) if bool(events & zmq.POLLIN): - print "here we go, nobody told me about new messages!" + print("here we go, nobody told me about new messages!") global STOP_EVERYTHING STOP_EVERYTHING = True raise gevent.GreenletExit() @@ -111,7 +113,7 @@ def server(): socket = ZMQSocket(zmq_context, zmq.REP) socket.bind('ipc://zmqbug') - class Cnt: + class Cnt(object): responded = 0 cnt = Cnt() @@ -125,15 +127,15 @@ def responder(): gevent.spawn(responder) while not STOP_EVERYTHING: - print "cnt.responded=", cnt.responded + print("cnt.responded=", cnt.responded) gevent.sleep(0.5) def client(): - socket = ZMQSocket(zmq_context, zmq.XREQ) + socket = ZMQSocket(zmq_context, zmq.DEALER) socket.connect('ipc://zmqbug') - class Cnt: + class Cnt(object): recv = 0 send = 0 @@ -156,7 +158,7 @@ def sendmsg(): gevent.spawn(sendmsg) while not STOP_EVERYTHING: - print "cnt.recv=", cnt.recv, "cnt.send=", cnt.send + print("cnt.recv=", cnt.recv, "cnt.send=", cnt.send) gevent.sleep(0.5) gevent.spawn(server) diff --git a/tox.ini b/tox.ini index a9a789f..96bace8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,16 @@ [tox] -envlist = py26,py27,py31,py32 +envlist = py26,py27,py34,py35 [testenv] -deps=nose -commands=nosetests +deps = + flake8 + pytest +commands = + flake8 zerorpc bin + pytest -v +passenv = ZPC_TEST_TIME_FACTOR + +[flake8] +ignore = E501,E128 +filename = *.py,zerorpc +exclude = tests,.git,dist,doc,*.egg-info,__pycache__,setup.py diff --git a/zerorpc/__init__.py b/zerorpc/__init__.py index 4e1b040..23e6894 100644 --- a/zerorpc/__init__.py +++ b/zerorpc/__init__.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,6 +22,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# Tell flake8 to ignore this file (otherwise it will complain about import *) +# flake8: noqa from .version import * from .exceptions import * from .context import * diff --git a/zerorpc/channel.py b/zerorpc/channel.py index 206fc4e..ad21c27 100644 --- a/zerorpc/channel.py +++ b/zerorpc/channel.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,59 +22,64 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import sys - import gevent.pool import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock +import logging from .exceptions import TimeoutExpired +from .channel_base import ChannelBase + + +logger = logging.getLogger(__name__) -class ChannelMultiplexer(object): +class ChannelMultiplexer(ChannelBase): def __init__(self, events, ignore_broadcast=False): self._events = events self._active_channels = {} self._channel_dispatcher_task = None self._broadcast_queue = None - if events.recv_is_available and not ignore_broadcast: + if events.recv_is_supported and not ignore_broadcast: self._broadcast_queue = gevent.queue.Queue(maxsize=1) self._channel_dispatcher_task = gevent.spawn( self._channel_dispatcher) @property - def recv_is_available(self): - return self._events.recv_is_available + def recv_is_supported(self): + return self._events.recv_is_supported - def __del__(self): - self.close() + @property + def emit_is_supported(self): + return self._events.emit_is_supported def close(self): if self._channel_dispatcher_task: self._channel_dispatcher_task.kill() - def create_event(self, name, args, xheader={}): - return self._events.create_event(name, args, xheader) - - def emit_event(self, event, identity=None): - return self._events.emit_event(event, identity) + def new_event(self, name, args, xheader=None): + return self._events.new_event(name, args, xheader) - def emit(self, name, args, xheader={}): - return self._events.emit(name, args, xheader) + def emit_event(self, event, timeout=None): + return self._events.emit_event(event, timeout) - def recv(self): + def recv(self, timeout=None): if self._broadcast_queue is not None: - event = self._broadcast_queue.get() + event = self._broadcast_queue.get(timeout=timeout) else: - event = self._events.recv() + event = self._events.recv(timeout=timeout) return event def _channel_dispatcher(self): while True: - event = self._events.recv() - channel_id = event.header.get('response_to', None) + try: + event = self._events.recv() + except Exception: + logger.exception('zerorpc.ChannelMultiplexer ignoring error on recv') + continue + channel_id = event.header.get(u'response_to', None) queue = None if channel_id is not None: @@ -85,10 +90,9 @@ def _channel_dispatcher(self): queue = self._broadcast_queue if queue is None: - print >> sys.stderr, \ - 'zerorpc.ChannelMultiplexer, ', \ - 'unable to route event:', \ - event.__str__(ignore_args=True) + logger.warning('zerorpc.ChannelMultiplexer,' + ' unable to route event: {0}'.format( + event.__str__(ignore_args=True))) else: queue.put(event) @@ -107,7 +111,7 @@ def context(self): return self._events.context -class Channel(object): +class Channel(ChannelBase): def __init__(self, multiplexer, from_event=None): self._multiplexer = multiplexer @@ -115,38 +119,39 @@ def __init__(self, multiplexer, from_event=None): self._zmqid = None self._queue = gevent.queue.Queue(maxsize=1) if from_event is not None: - self._channel_id = from_event.header['message_id'] - self._zmqid = from_event.header.get('zmqid', None) + self._channel_id = from_event.header[u'message_id'] + self._zmqid = from_event.identity self._multiplexer._active_channels[self._channel_id] = self + logger.debug('<-- new channel %s', self._channel_id) self._queue.put(from_event) @property - def recv_is_available(self): - return self._multiplexer.recv_is_available + def recv_is_supported(self): + return self._multiplexer.recv_is_supported - def __del__(self): - self.close() + @property + def emit_is_supported(self): + return self._multiplexer.emit_is_supported def close(self): if self._channel_id is not None: del self._multiplexer._active_channels[self._channel_id] + logger.debug('-x- closed channel %s', self._channel_id) self._channel_id = None - def create_event(self, name, args, xheader={}): - event = self._multiplexer.create_event(name, args, xheader) + def new_event(self, name, args, xheader=None): + event = self._multiplexer.new_event(name, args, xheader) if self._channel_id is None: - self._channel_id = event.header['message_id'] + self._channel_id = event.header[u'message_id'] self._multiplexer._active_channels[self._channel_id] = self + logger.debug('--> new channel %s', self._channel_id) else: - event.header['response_to'] = self._channel_id + event.header[u'response_to'] = self._channel_id + event.identity = self._zmqid return event - def emit(self, name, args, xheader={}): - event = self.create_event(name, args, xheader) - self._multiplexer.emit_event(event, self._zmqid) - - def emit_event(self, event): - self._multiplexer.emit_event(event, self._zmqid) + def emit_event(self, event, timeout=None): + self._multiplexer.emit_event(event, timeout) def recv(self, timeout=None): try: @@ -160,7 +165,7 @@ def context(self): return self._multiplexer.context -class BufferedChannel(object): +class BufferedChannel(ChannelBase): def __init__(self, channel, inqueue_size=100): self._channel = channel @@ -169,14 +174,17 @@ def __init__(self, channel, inqueue_size=100): self._input_queue_reserved = 1 self._remote_can_recv = gevent.event.Event() self._input_queue = gevent.queue.Queue() - self._lost_remote = False self._verbose = False self._on_close_if = None self._recv_task = gevent.spawn(self._recver) @property - def recv_is_available(self): - return self._channel.recv_is_available + def recv_is_supported(self): + return self._channel.recv_is_supported + + @property + def emit_is_supported(self): + return self._channel.emit_is_supported @property def on_close_if(self): @@ -186,9 +194,6 @@ def on_close_if(self): def on_close_if(self, cb): self._on_close_if = cb - def __del__(self): - self.close() - def close(self): if self._recv_task is not None: self._recv_task.kill() @@ -200,18 +205,16 @@ def close(self): def _recver(self): while True: event = self._channel.recv() - if event.name == '_zpc_more': + if event.name == u'_zpc_more': try: self._remote_queue_open_slots += int(event.args[0]) - except Exception as e: - print >> sys.stderr, \ - 'gevent_zerorpc.BufferedChannel._recver,', \ - 'exception:', e + except Exception: + logger.exception('gevent_zerorpc.BufferedChannel._recver') if self._remote_queue_open_slots > 0: self._remote_can_recv.set() elif self._input_queue.qsize() == self._input_queue_size: raise RuntimeError( - 'BufferedChannel, queue overflow on event:', event) + 'BufferedChannel, queue overflow on event:', event) else: self._input_queue.put(event) if self._on_close_if is not None and self._on_close_if(event): @@ -219,13 +222,11 @@ def _recver(self): self.close() return - def create_event(self, name, args, xheader={}): - return self._channel.create_event(name, args, xheader) + def new_event(self, name, args, xheader=None): + return self._channel.new_event(name, args, xheader) - def emit_event(self, event, block=True, timeout=None): + def emit_event(self, event, timeout=None): if self._remote_queue_open_slots == 0: - if not block: - return False self._remote_can_recv.clear() self._remote_can_recv.wait(timeout=timeout) self._remote_queue_open_slots -= 1 @@ -234,20 +235,18 @@ def emit_event(self, event, block=True, timeout=None): except: self._remote_queue_open_slots += 1 raise - return True - - def emit(self, name, args, xheader={}, block=True, timeout=None): - event = self.create_event(name, args, xheader) - return self.emit_event(event, block, timeout) def _request_data(self): open_slots = self._input_queue_size - self._input_queue_reserved self._input_queue_reserved += open_slots - self._channel.emit('_zpc_more', (open_slots,)) + self._channel.emit(u'_zpc_more', (open_slots,)) def recv(self, timeout=None): - if self._verbose: - if self._input_queue_reserved < self._input_queue_size / 2: + # self._channel can be set to None by an 'on_close_if' callback if it + # sees a suitable message from the remote end... + # + if self._verbose and self._channel: + if self._input_queue_reserved < self._input_queue_size // 2: self._request_data() else: self._verbose = True diff --git a/zerorpc/channel_base.py b/zerorpc/channel_base.py new file mode 100644 index 0000000..a391b08 --- /dev/null +++ b/zerorpc/channel_base.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Open Source Initiative OSI - The MIT License (MIT):Licensing +# +# The MIT License (MIT) +# Copyright (c) 2014 François-Xavier Bourlet (bombela@gmail.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +class ChannelBase(object): + + @property + def context(self): + raise NotImplementedError() + + @property + def recv_is_supported(self): + raise NotImplementedError() + + @property + def emit_is_supported(self): + raise NotImplementedError() + + def close(self): + raise NotImplementedError() + + def new_event(self, name, args, xheader=None): + raise NotImplementedError() + + def emit_event(self, event, timeout=None): + raise NotImplementedError() + + def emit(self, name, args, xheader=None, timeout=None): + event = self.new_event(name, args, xheader) + return self.emit_event(event, timeout) + + def recv(self, timeout=None): + raise NotImplementedError() diff --git a/zerorpc/cli.py b/zerorpc/cli.py new file mode 100644 index 0000000..3fd6350 --- /dev/null +++ b/zerorpc/cli.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Open Source Initiative OSI - The MIT License (MIT):Licensing +# +# The MIT License (MIT) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +from __future__ import print_function +from builtins import map + +import argparse +import json +import sys +import inspect +import os +import logging +from pprint import pprint + +import zerorpc + +try: + # Try collections.abc for 3.4+ + from collections.abc import Iterator +except ImportError: + # Fallback to collections for Python 2 + from collections import Iterator + + +parser = argparse.ArgumentParser( + description='Make a zerorpc call to a remote service.' +) + +client_or_server = parser.add_mutually_exclusive_group() +client_or_server.add_argument('--client', action='store_true', default=True, + help='remote procedure call mode (default)') +client_or_server.add_argument('--server', action='store_false', dest='client', + help='turn a given python module into a server') + +parser.add_argument('--connect', action='append', metavar='address', + help='specify address to connect to. Can be specified \ + multiple times and in conjunction with --bind') +parser.add_argument('--bind', action='append', metavar='address', + help='specify address to listen to. Can be specified \ + multiple times and in conjunction with --connect') +parser.add_argument('--timeout', default=30, metavar='seconds', type=int, + help='abort request after X seconds. \ + (default: 30s, --client only)') +parser.add_argument('--heartbeat', default=5, metavar='seconds', type=int, + help='heartbeat frequency. You should always use \ + the same frequency as the server. (default: 5s)') +parser.add_argument('--pool-size', default=None, metavar='count', type=int, + help='size of worker pool. --server only.') +parser.add_argument('-j', '--json', default=False, action='store_true', + help='arguments are in JSON format and will be be parsed \ + before being sent to the remote') +parser.add_argument('-pj', '--print-json', default=False, action='store_true', + help='print result in JSON format.') +parser.add_argument('-?', '--inspect', default=False, action='store_true', + help='retrieve detailed informations for the given \ + remote (cf: command) method. If not method, display \ + a list of remote methods signature. (only for --client).') +parser.add_argument('--active-hb', default=False, action='store_true', + help='enable active heartbeat. The default is to \ + wait for the server to send the first heartbeat') +parser.add_argument('-d', '--debug', default=False, action='store_true', + help='Print zerorpc debug msgs, \ + like outgoing and incomming messages.') +parser.add_argument('address', nargs='?', help='address to connect to. Skip \ + this if you specified --connect or --bind at least once') +parser.add_argument('command', nargs='?', + help='remote procedure to call if --client (default) or \ + python module/class to load if --server. If no command is \ + specified, a list of remote methods are displayed.') +parser.add_argument('params', nargs='*', + help='parameters for the remote call if --client \ + (default)') + + +def setup_links(args, socket): + if args.bind: + for endpoint in args.bind: + print('binding to "{0}"'.format(endpoint), file=sys.stderr) + socket.bind(endpoint) + addresses = [] + if args.address: + addresses.append(args.address) + if args.connect: + addresses.extend(args.connect) + for endpoint in addresses: + print('connecting to "{0}"'.format(endpoint), file=sys.stderr) + socket.connect(endpoint) + + +def run_server(args): + server_obj_path = args.command + + sys.path.insert(0, os.getcwd()) + if '.' in server_obj_path: + modulepath, objname = server_obj_path.rsplit('.', 1) + module = __import__(modulepath, fromlist=[objname]) + server_obj = getattr(module, objname) + else: + server_obj = __import__(server_obj_path) + + if callable(server_obj): + server_obj = server_obj() + + server = zerorpc.Server(server_obj, heartbeat=args.heartbeat, pool_size=args.pool_size) + if args.debug: + server.debug = True + setup_links(args, server) + print('serving "{0}"'.format(server_obj_path), file=sys.stderr) + return server.run() + + +# this function does a really intricate job to keep backward compatibility +# with a previous version of zerorpc, and lazily retrieving results if possible +def zerorpc_inspect_legacy(client, filter_method, long_doc, include_argspec): + if filter_method is None: + remote_methods = client._zerorpc_list() + else: + remote_methods = [filter_method] + + def remote_detailled_methods(): + for name in remote_methods: + if include_argspec: + argspec = client._zerorpc_args(name) + else: + argspec = None + docstring = client._zerorpc_help(name) + if docstring and not long_doc: + docstring = docstring.split('\n', 1)[0] + yield (name, argspec, docstring if docstring else '') + + if not include_argspec: + longest_name_len = max(len(name) for name in remote_methods) + return (longest_name_len, ((name, doc) for name, argspec, doc in + remote_detailled_methods())) + + r = [(name + (inspect.formatargspec(*argspec) + if argspec else '(...)'), doc) + for name, argspec, doc in remote_detailled_methods()] + longest_name_len = max(len(name) for name, doc in r) if r else 0 + return (longest_name_len, r) + + +# handle the 'python formatted' _zerorpc_inspect, that return the output of +# "getargspec" from the python lib "inspect". A monstruosity from protocol v2. +def zerorpc_inspect_python_argspecs(remote_methods, filter_method, long_doc, include_argspec): + def format_method(name, argspec, doc): + if include_argspec: + name += (inspect.formatargspec(*argspec) if argspec else + '(...)') + if not doc: + doc = '' + elif not long_doc: + doc = doc.splitlines()[0] + return (name, doc) + r = [format_method(*methods_info) for methods_info in remote_methods if + filter_method is None or methods_info[0] == filter_method] + if not r: + return None + longest_name_len = max(len(name) for name, doc in r) if r else 0 + return (longest_name_len, r) + + +# Handles generically formatted arguments (not tied to any specific programming language). +def zerorpc_inspect_generic(remote_methods, filter_method, long_doc, include_argspec): + def format_method(name, args, doc): + if include_argspec: + def format_arg(arg): + def_val = arg.get('default') + if def_val is None: + return arg['name'] + return '{0}={1}'.format(arg['name'], def_val) + + if args: + name += '({0})'.format(', '.join(map(format_arg, args))) + else: + name += '(??)' + + if not doc: + doc = '' + elif not long_doc: + doc = doc.splitlines()[0] + return (name, doc) + + methods = [format_method(name, details['args'], details['doc']) + for name, details in remote_methods.items() + if filter_method is None or name == filter_method] + + longest_name_len = (max(len(name) for name, doc in methods) + if methods else 0) + return (longest_name_len, methods) + + +def zerorpc_inspect(client, method=None, long_doc=True, include_argspec=True): + try: + inspect_result = client._zerorpc_inspect() + remote_methods = inspect_result['methods'] + legacy = False + except (zerorpc.RemoteError, NameError): + legacy = True + + if legacy: + try: + service_name = client._zerorpc_name() + except (zerorpc.RemoteError): + service_name = 'N/A' + + (longest_name_len, detailled_methods) = zerorpc_inspect_legacy(client, + method, long_doc, include_argspec) + else: + service_name = inspect_result.get('name', 'N/A') + if not isinstance(remote_methods, dict): + (longest_name_len, + detailled_methods) = zerorpc_inspect_python_argspecs( + remote_methods, method, long_doc, include_argspec) + + (longest_name_len, detailled_methods) = zerorpc_inspect_generic( + remote_methods, method, long_doc, include_argspec) + + return longest_name_len, detailled_methods, service_name + + +def run_client(args): + client = zerorpc.Client(timeout=args.timeout, heartbeat=args.heartbeat, + passive_heartbeat=not args.active_hb) + if args.debug: + client.debug = True + setup_links(args, client) + if not args.command: + (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, + long_doc=False, include_argspec=args.inspect) + print('[{0}]'.format(service)) + if args.inspect: + for (name, doc) in detailled_methods: + print(name) + else: + for (name, doc) in detailled_methods: + print('{0} {1}'.format(name.ljust(longest_name_len), doc)) + return + if args.inspect: + (longest_name_len, detailled_methods, service) = zerorpc_inspect(client, + method=args.command) + if detailled_methods: + (name, doc) = detailled_methods[0] + print('[{0}]\n{1}\n\n{2}\n'.format(service, name, doc)) + else: + print('[{0}]\nNo documentation for "{1}".'.format(service, args.command)) + return + if args.json: + call_args = [json.loads(x) for x in args.params] + else: + call_args = args.params + results = client(args.command, *call_args) + if not isinstance(results, Iterator): + if args.print_json: + json.dump(results, sys.stdout) + else: + pprint(results) + else: + # streaming responses + if args.print_json: + first = True + sys.stdout.write('[') + for result in results: + if first: + first = False + else: + sys.stdout.write(',') + json.dump(result, sys.stdout) + sys.stdout.write(']') + else: + for result in results: + pprint(result) + + +def main(): + logging.basicConfig() + args = parser.parse_args() + + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + + if args.bind or args.connect: + if args.command: + args.params.insert(0, args.command) + args.command = args.address + args.address = None + + if not (args.bind or args.connect or args.address): + parser.print_help() + return -1 + + if args.client: + return run_client(args) + + if not args.command: + parser.print_help() + return -1 + + return run_server(args) diff --git a/zerorpc/context.py b/zerorpc/context.py index 4578ba7..debce26 100644 --- a/zerorpc/context.py +++ b/zerorpc/context.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,28 +23,79 @@ # SOFTWARE. +from __future__ import absolute_import +from future.utils import tobytes + import uuid -import functools import random -import gevent_zmq as zmq +from . import gevent_zmq as zmq class Context(zmq.Context): _instance = None def __init__(self): + super(zmq.Context, self).__init__() self._middlewares = [] - self._middlewares_hooks = { + self._hooks = { 'resolve_endpoint': [], - 'raise_error': [], - 'call_procedure': [], 'load_task_context': [], 'get_task_context': [], - 'inspect_error': [] + 'server_before_exec': [], + 'server_after_exec': [], + 'server_inspect_exception': [], + 'client_handle_remote_error': [], + 'client_before_request': [], + 'client_after_request': [], + 'client_patterns_list': [], } self._reset_msgid() + # NOTE: pyzmq 13.0.0 messed up with setattr (they turned it into a + # non-op) and you can't assign attributes normally anymore, hence the + # tricks with self.__dict__ here + + @property + def _middlewares(self): + return self.__dict__['_middlewares'] + + @_middlewares.setter + def _middlewares(self, value): + self.__dict__['_middlewares'] = value + + @property + def _hooks(self): + return self.__dict__['_hooks'] + + @_hooks.setter + def _hooks(self, value): + self.__dict__['_hooks'] = value + + @property + def _msg_id_base(self): + return self.__dict__['_msg_id_base'] + + @_msg_id_base.setter + def _msg_id_base(self, value): + self.__dict__['_msg_id_base'] = value + + @property + def _msg_id_counter(self): + return self.__dict__['_msg_id_counter'] + + @_msg_id_counter.setter + def _msg_id_counter(self, value): + self.__dict__['_msg_id_counter'] = value + + @property + def _msg_id_counter_stop(self): + return self.__dict__['_msg_id_counter_stop'] + + @_msg_id_counter_stop.setter + def _msg_id_counter_stop(self, value): + self.__dict__['_msg_id_counter_stop'] = value + @staticmethod def get_instance(): if Context._instance is None: @@ -52,21 +103,21 @@ def get_instance(): return Context._instance def _reset_msgid(self): - self._msg_id_base = str(uuid.uuid4())[8:] - self._msg_id_counter = random.randrange(0, 2**32) - self._msg_id_counter_stop = random.randrange(self._msg_id_counter, 2**32) + self._msg_id_base = tobytes(uuid.uuid4().hex)[8:] + self._msg_id_counter = random.randrange(0, 2 ** 32) + self._msg_id_counter_stop = random.randrange(self._msg_id_counter, 2 ** 32) def new_msgid(self): - if (self._msg_id_counter >= self._msg_id_counter_stop): + if self._msg_id_counter >= self._msg_id_counter_stop: self._reset_msgid() else: - self._msg_id_counter = (self._msg_id_counter + 1) & 0xffffffff - return '{0:08x}{1}'.format(self._msg_id_counter, self._msg_id_base) + self._msg_id_counter = (self._msg_id_counter + 1) + return tobytes('{0:08x}'.format(self._msg_id_counter)) + self._msg_id_base def register_middleware(self, middleware_instance): registered_count = 0 self._middlewares.append(middleware_instance) - for hook in self._middlewares_hooks.keys(): + for hook in self._hooks: functor = getattr(middleware_instance, hook, None) if functor is None: try: @@ -74,45 +125,104 @@ def register_middleware(self, middleware_instance): except AttributeError: pass if functor is not None: - self._middlewares_hooks[hook].append(functor) + self._hooks[hook].append(functor) registered_count += 1 return registered_count - def middleware_resolve_endpoint(self, endpoint): - for functor in self._middlewares_hooks['resolve_endpoint']: + # + # client/server + # + def hook_resolve_endpoint(self, endpoint): + for functor in self._hooks['resolve_endpoint']: endpoint = functor(endpoint) return endpoint - def middleware_inspect_error(self, exc_type, exc_value, exc_traceback): - exc_info = exc_type, exc_value, exc_traceback - task_context = self.middleware_get_task_context() - for functor in self._middlewares_hooks['inspect_error']: - functor(task_context, exc_info) + def hook_load_task_context(self, event_header): + for functor in self._hooks['load_task_context']: + functor(event_header) + + def hook_get_task_context(self): + event_header = {} + for functor in self._hooks['get_task_context']: + event_header.update(functor()) + return event_header + + # + # Server-side hooks + # + def hook_server_before_exec(self, request_event): + """Called when a method is about to be executed on the server.""" - def middleware_raise_error(self, event): - for functor in self._middlewares_hooks['raise_error']: + for functor in self._hooks['server_before_exec']: + functor(request_event) + + def hook_server_after_exec(self, request_event, reply_event): + """Called when a method has been executed successfully. + + This hook is called right before the answer is sent back to the client. + If the method streams its answer (i.e: it uses the zerorpc.stream + decorator) then this hook will be called once the reply has been fully + streamed (and right before the stream is "closed"). + + The reply_event argument will be None if the Push/Pull pattern is used. + + """ + for functor in self._hooks['server_after_exec']: + functor(request_event, reply_event) + + def hook_server_inspect_exception(self, request_event, reply_event, exc_infos): + """Called when a method raised an exception. + + The reply_event argument will be None if the Push/Pull pattern is used. + + """ + task_context = self.hook_get_task_context() + for functor in self._hooks['server_inspect_exception']: + functor(request_event, reply_event, task_context, exc_infos) + + # + # Client-side hooks + # + def hook_client_handle_remote_error(self, event): + exception = None + for functor in self._hooks['client_handle_remote_error']: + ret = functor(event) + if ret: + exception = ret + return exception + + def hook_client_before_request(self, event): + """Called when the Client is about to send a request. + + You can see it as the counterpart of ``hook_server_before_exec``. + + """ + for functor in self._hooks['client_before_request']: functor(event) - def middleware_call_procedure(self, procedure, *args, **kwargs): - class chain(object): - def __init__(self, fct, next): - functools.update_wrapper(self, next) - self.fct = fct - self.next = next + def hook_client_after_request(self, request_event, reply_event, exception=None): + """Called when an answer or a timeout has been received from the server. - def __call__(self, *args, **kwargs): - return self.fct(self.next, *args, **kwargs) + This hook is called right before the answer is returned to the client. + You can see it as the counterpart of the ``hook_server_after_exec``. - for functor in self._middlewares_hooks['call_procedure']: - procedure = chain(functor, procedure) - return procedure(*args, **kwargs) + If the called method was returning a stream (i.e: it uses the + zerorpc.stream decorator) then this hook will be called once the reply + has been fully streamed (when the stream is "closed") or when an + exception has been raised. - def middleware_load_task_context(self, event_header): - for functor in self._middlewares_hooks['load_task_context']: - functor(event_header) + The optional exception argument will be a ``RemoteError`` (or whatever + type returned by the client_handle_remote_error hook) if an exception + has been raised on the server. - def middleware_get_task_context(self): - event_header = {} - for functor in self._middlewares_hooks['get_task_context']: - event_header.update(functor()) - return event_header + If the request timed out, then the exception argument will be a + ``TimeoutExpired`` object and reply_event will be None. + + """ + for functor in self._hooks['client_after_request']: + functor(request_event, reply_event, exception) + + def hook_client_patterns_list(self, patterns): + for functor in self._hooks['client_patterns_list']: + patterns = functor(patterns) + return patterns diff --git a/zerorpc/core.py b/zerorpc/core.py index 73538d3..ec2e008 100644 --- a/zerorpc/core.py +++ b/zerorpc/core.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,22 +23,31 @@ # SOFTWARE. +from __future__ import absolute_import +from builtins import str +from builtins import zip +from future.utils import iteritems + import sys import traceback import gevent.pool import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock -import gevent_zmq as zmq +from . import gevent_zmq as zmq from .exceptions import TimeoutExpired, RemoteError, LostRemote from .channel import ChannelMultiplexer, BufferedChannel from .socket import SocketBase from .heartbeat import HeartBeatOnChannel from .context import Context from .decorators import DecoratorBase, rep -import patterns +from . import patterns +from logging import getLogger + +logger = getLogger(__name__) + class ServerBase(object): @@ -50,45 +59,54 @@ def __init__(self, channel, methods=None, name=None, context=None, methods = self self._context = context or Context.get_instance() - self._name = name or repr(methods) + self._name = name or self._extract_name() self._task_pool = gevent.pool.Pool(size=pool_size) self._acceptor_task = None - self._methods = self._zerorpc_filter_methods(ServerBase, self, methods) + self._methods = self._filter_methods(ServerBase, self, methods) self._inject_builtins() self._heartbeat_freq = heartbeat - for (k, functor) in self._methods.items(): + for (k, functor) in iteritems(self._methods): if not isinstance(functor, DecoratorBase): self._methods[k] = rep(functor) @staticmethod - def _zerorpc_filter_methods(cls, self, methods): - if hasattr(methods, '__getitem__'): + def _filter_methods(cls, self, methods): + if isinstance(methods, dict): return methods - server_methods = set(getattr(self, k) for k in dir(cls) if not - k.startswith('_')) + server_methods = set(k for k in dir(cls) if not k.startswith('_')) return dict((k, getattr(methods, k)) - for k in dir(methods) - if callable(getattr(methods, k)) - and not k.startswith('_') - and getattr(methods, k) not in server_methods - ) + for k in dir(methods) + if callable(getattr(methods, k)) and + not k.startswith('_') and k not in server_methods + ) + + @staticmethod + def _extract_name(methods): + return getattr(methods, '__name__', None) \ + or getattr(type(methods), '__name__', None) \ + or repr(methods) def close(self): self.stop() self._multiplexer.close() - def _zerorpc_inspect(self, method=None, long_doc=True): - if method: - methods = {method: self._methods[method]} - else: - methods = dict((m, f) for m, f in self._methods.items() + def _format_args_spec(self, args_spec, r=None): + if args_spec: + r = [dict(name=name) for name in args_spec[0]] + default_values = args_spec[3] + if default_values is not None: + for arg, def_val in zip(reversed(r), reversed(default_values)): + arg['default'] = def_val + return r + + def _zerorpc_inspect(self): + methods = dict((m, f) for m, f in iteritems(self._methods) if not m.startswith('_')) - detailled_methods = [(m, f._zerorpc_args(), - f.__doc__ if long_doc else - f.__doc__.split('\n', 1)[0] if f.__doc__ else None) - for (m, f) in methods.items()] + detailled_methods = dict((m, + dict(args=self._format_args_spec(f._zerorpc_args()), + doc=f._zerorpc_doc())) for (m, f) in iteritems(methods)) return {'name': self._name, 'methods': detailled_methods} @@ -97,7 +115,8 @@ def _inject_builtins(self): if not m.startswith('_')] self._methods['_zerorpc_name'] = lambda: self._name self._methods['_zerorpc_ping'] = lambda: ['pong', self._name] - self._methods['_zerorpc_help'] = lambda m: self._methods[m].__doc__ + self._methods['_zerorpc_help'] = lambda m: \ + self._methods[m]._zerorpc_doc() self._methods['_zerorpc_args'] = \ lambda m: self._methods[m]._zerorpc_args() self._methods['_zerorpc_inspect'] = self._zerorpc_inspect @@ -107,45 +126,43 @@ def __call__(self, method, *args): raise NameError(method) return self._methods[method](*args) - def _print_traceback(self, protocol_v1): - exc_type, exc_value, exc_traceback = sys.exc_info() - try: - traceback.print_exception(exc_type, exc_value, exc_traceback, - file=sys.stderr) + def _print_traceback(self, protocol_v1, exc_infos): + logger.exception('') - self._context.middleware_inspect_error(exc_type, exc_value, - exc_traceback) - - if protocol_v1: - return (repr(exc_value),) - - human_traceback = traceback.format_exc() - name = exc_type.__name__ - human_msg = str(exc_value) - return (name, human_msg, human_traceback) - finally: - del exc_traceback + exc_type, exc_value, exc_traceback = exc_infos + if protocol_v1: + return (repr(exc_value),) + human_traceback = traceback.format_exc() + name = exc_type.__name__ + human_msg = str(exc_value) + return (name, human_msg, human_traceback) def _async_task(self, initial_event): - protocol_v1 = initial_event.header.get('v', 1) < 2 + protocol_v1 = initial_event.header.get(u'v', 1) < 2 channel = self._multiplexer.channel(initial_event) hbchan = HeartBeatOnChannel(channel, freq=self._heartbeat_freq, passive=protocol_v1) bufchan = BufferedChannel(hbchan) + exc_infos = None event = bufchan.recv() try: - self._context.middleware_load_task_context(event.header) + self._context.hook_load_task_context(event.header) functor = self._methods.get(event.name, None) if functor is None: raise NameError(event.name) functor.pattern.process_call(self._context, bufchan, event, functor) except LostRemote: - self._print_traceback(protocol_v1) + exc_infos = list(sys.exc_info()) + self._print_traceback(protocol_v1, exc_infos) except Exception: - exception_info = self._print_traceback(protocol_v1) - bufchan.emit('ERR', exception_info, - self._context.middleware_get_task_context()) + exc_infos = list(sys.exc_info()) + human_exc_infos = self._print_traceback(protocol_v1, exc_infos) + reply_event = bufchan.new_event(u'ERR', human_exc_infos, + self._context.hook_get_task_context()) + self._context.hook_server_inspect_exception(event, reply_event, exc_infos) + bufchan.emit_event(reply_event) finally: + del exc_infos bufchan.close() def _acceptor(self): @@ -181,59 +198,84 @@ def __init__(self, channel, context=None, timeout=30, heartbeat=5, def close(self): self._multiplexer.close() - def _raise_remote_error(self, event): - self._context.middleware_raise_error(event) + def _handle_remote_error(self, event): + exception = self._context.hook_client_handle_remote_error(event) + if not exception: + if event.header.get(u'v', 1) >= 2: + (name, msg, traceback) = event.args + exception = RemoteError(name, msg, traceback) + else: + (msg,) = event.args + exception = RemoteError('RemoteError', msg, None) - if event.header.get('v', 1) >= 2: - (name, msg, traceback) = event.args - raise RemoteError(name, msg, traceback) - else: - (msg,) = event.args - raise RemoteError('RemoteError', msg, None) + return exception def _select_pattern(self, event): - for pattern in patterns.patterns_list: + for pattern in self._context.hook_client_patterns_list( + patterns.patterns_list): if pattern.accept_answer(event): return pattern - msg = 'Unable to find a pattern for: {0}'.format(event) - raise RuntimeError(msg) + return None - def _process_response(self, method, bufchan, timeout): - try: - try: - event = bufchan.recv(timeout) - except TimeoutExpired: - raise TimeoutExpired(timeout, - 'calling remote method {0}'.format(method)) - - pattern = self._select_pattern(event) - return pattern.process_answer(self._context, bufchan, event, method, - self._raise_remote_error) - except: + def _process_response(self, request_event, bufchan, timeout): + def raise_error(ex): bufchan.close() - raise + self._context.hook_client_after_request(request_event, None, ex) + raise ex + + try: + reply_event = bufchan.recv(timeout=timeout) + except TimeoutExpired: + raise_error(TimeoutExpired(timeout, + 'calling remote method {0}'.format(request_event.name))) + + pattern = self._select_pattern(reply_event) + if pattern is None: + raise_error(RuntimeError( + 'Unable to find a pattern for: {0}'.format(request_event))) + + return pattern.process_answer(self._context, bufchan, request_event, + reply_event, self._handle_remote_error) def __call__(self, method, *args, **kargs): + # here `method` is either a string of bytes or an unicode string in + # Python2 and Python3. Python2: str aka a byte string containing ASCII + # (unless the user explicitly provide an unicode string). Python3: str + # aka an unicode string (unless the user explicitly provide a byte + # string). + # zerorpc protocol requires an utf-8 encoded string at the msgpack + # level. msgpack will encode any unicode string object to UTF-8 and tag + # it `string`, while a bytes string will be tagged `bin`. + # + # So when we get a bytes string, we assume it to be an UTF-8 string + # (ASCII is contained in UTF-8) that we decode to an unicode string. + # Right after, msgpack-python will re-encode it as UTF-8. Yes this is + # terribly inefficient with Python2 because most of the time `method` + # will already be an UTF-8 encoded bytes string. + if isinstance(method, bytes): + method = method.decode('utf-8') + timeout = kargs.get('timeout', self._timeout) channel = self._multiplexer.channel() hbchan = HeartBeatOnChannel(channel, freq=self._heartbeat_freq, passive=self._passive_heartbeat) bufchan = BufferedChannel(hbchan, inqueue_size=kargs.get('slots', 100)) - xheader = self._context.middleware_get_task_context() - bufchan.emit(method, args, xheader) + xheader = self._context.hook_get_task_context() + request_event = bufchan.new_event(method, args, xheader) + self._context.hook_client_before_request(request_event) + bufchan.emit_event(request_event) - try: - if kargs.get('async', False) is False: - return self._process_response(method, bufchan, timeout) - - async_result = gevent.event.AsyncResult() - gevent.spawn(self._process_response, method, bufchan, - timeout).link(async_result) - return async_result - except: - bufchan.close() - raise + # In python 3.7, "async" is a reserved keyword, clients should now use + # "async_": support both for the time being + if (kargs.get('async', False) is False and + kargs.get('async_', False) is False): + return self._process_response(request_event, bufchan, timeout) + + async_result = gevent.event.AsyncResult() + gevent.spawn(self._process_response, request_event, bufchan, + timeout).link(async_result) + return async_result def __getattr__(self, method): return lambda *args, **kargs: self(method, *args, **kargs) @@ -243,10 +285,12 @@ class Server(SocketBase, ServerBase): def __init__(self, methods=None, name=None, context=None, pool_size=None, heartbeat=5): - SocketBase.__init__(self, zmq.XREP, context) + SocketBase.__init__(self, zmq.ROUTER, context) if methods is None: methods = self - methods = ServerBase._zerorpc_filter_methods(Server, self, methods) + + name = name or ServerBase._extract_name(methods) + methods = ServerBase._filter_methods(Server, self, methods) ServerBase.__init__(self, self._events, methods, name, context, pool_size, heartbeat) @@ -259,7 +303,7 @@ class Client(SocketBase, ClientBase): def __init__(self, connect_to=None, context=None, timeout=30, heartbeat=5, passive_heartbeat=False): - SocketBase.__init__(self, zmq.XREQ, context=context) + SocketBase.__init__(self, zmq.DEALER, context=context) ClientBase.__init__(self, self._events, context, timeout, heartbeat, passive_heartbeat) if connect_to: @@ -277,7 +321,7 @@ def __init__(self, context=None, zmq_socket=zmq.PUSH): def __call__(self, method, *args): self._events.emit(method, args, - self._context.middleware_get_task_context()) + self._context.hook_get_task_context()) def __getattr__(self, method): return lambda *args: self(method, *args) @@ -291,7 +335,7 @@ def __init__(self, methods=None, context=None, zmq_socket=zmq.PULL): if methods is None: methods = self - self._methods = ServerBase._zerorpc_filter_methods(Puller, self, methods) + self._methods = ServerBase._filter_methods(Puller, self, methods) self._receiver_task = None def close(self): @@ -309,19 +353,19 @@ def _receiver(self): try: if event.name not in self._methods: raise NameError(event.name) - self._context.middleware_load_task_context(event.header) - self._context.middleware_call_procedure( - self._methods[event.name], - *event.args) + self._context.hook_load_task_context(event.header) + self._context.hook_server_before_exec(event) + self._methods[event.name](*event.args) + # In Push/Pull their is no reply to send, hence None for the + # reply_event argument + self._context.hook_server_after_exec(event, None) except Exception: - exc_type, exc_value, exc_traceback = sys.exc_info() + exc_infos = sys.exc_info() try: - traceback.print_exception(exc_type, exc_value, exc_traceback, - file=sys.stderr) - self._context.middleware_inspect_error(exc_type, exc_value, - exc_traceback) + logger.exception('') + self._context.hook_server_inspect_exception(event, None, exc_infos) finally: - del exc_traceback + del exc_infos def run(self): self._receiver_task = gevent.spawn(self._receiver) @@ -346,7 +390,7 @@ class Subscriber(Puller): def __init__(self, methods=None, context=None): super(Subscriber, self).__init__(methods=methods, context=context, zmq_socket=zmq.SUB) - self._events.setsockopt(zmq.SUBSCRIBE, '') + self._events.setsockopt(zmq.SUBSCRIBE, b'') def fork_task_context(functor, context=None): @@ -362,7 +406,7 @@ def fork_task_context(functor, context=None): - task1 is created to handle this event this task will be linked to the initial event context. zerorpc.Server does that for you. - task1 make use of some zerorpc.Client instances, the initial - event context is transfered on every call. + event context is transferred on every call. - task1 spawn a new task2. - task2 make use of some zerorpc.Client instances, it's a fresh @@ -371,7 +415,7 @@ def fork_task_context(functor, context=None): - task1 spawn a new fork_task_context(task3). - task3 make use of some zerorpc.Client instances, the initial - event context is transfered on every call. + event context is transferred on every call. A real use case is a distributed tracer. Each time a new event is created, a trace_id is injected in it or copied from the current task @@ -382,8 +426,9 @@ def fork_task_context(functor, context=None): - if the new task will make any zerorpc call, it should be wrapped. ''' context = context or Context.get_instance() - header = context.middleware_get_task_context() + xheader = context.hook_get_task_context() + def wrapped(*args, **kargs): - context.middleware_load_task_context(header) + context.hook_load_task_context(xheader) return functor(*args, **kargs) return wrapped diff --git a/zerorpc/decorators.py b/zerorpc/decorators.py index 0d785b7..43dfa64 100644 --- a/zerorpc/decorators.py +++ b/zerorpc/decorators.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -24,7 +24,7 @@ import inspect -from .patterns import * +from .patterns import ReqRep, ReqStream class DecoratorBase(object): @@ -33,7 +33,7 @@ class DecoratorBase(object): def __init__(self, functor): self._functor = functor self.__doc__ = functor.__doc__ - self.__name__ = functor.__name__ + self.__name__ = getattr(functor, "__name__", str(functor)) def __get__(self, instance, type_instance=None): if instance is None: @@ -43,6 +43,11 @@ def __get__(self, instance, type_instance=None): def __call__(self, *args, **kargs): return self._functor(*args, **kargs) + def _zerorpc_doc(self): + if self.__doc__ is None: + return None + return inspect.cleandoc(self.__doc__) + def _zerorpc_args(self): try: args_spec = self._functor._zerorpc_args() diff --git a/zerorpc/events.py b/zerorpc/events.py index 2044eef..f87d0b5 100644 --- a/zerorpc/events.py +++ b/zerorpc/events.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,103 +23,161 @@ # SOFTWARE. +from __future__ import absolute_import +from builtins import str +from builtins import range + import msgpack import gevent.pool import gevent.queue import gevent.event import gevent.local -import gevent.coros - +import gevent.lock +import logging +import sys -import gevent_zmq as zmq +from . import gevent_zmq as zmq +from .exceptions import TimeoutExpired from .context import Context +from .channel_base import ChannelBase + + +if sys.version_info < (2, 7): + def get_pyzmq_frame_buffer(frame): + return frame.buffer[:] +else: + def get_pyzmq_frame_buffer(frame): + return frame.buffer + +# gevent <= 1.1.0.rc5 is missing the Python3 __next__ method. +if sys.version_info >= (3, 0) and gevent.version_info <= (1, 1, 0, 'rc', '5'): + setattr(gevent.queue.Channel, '__next__', gevent.queue.Channel.next) -class Sender(object): +logger = logging.getLogger(__name__) + + +class SequentialSender(object): def __init__(self, socket): self._socket = socket - self._send_queue = gevent.queue.Queue(maxsize=0) - self._send_task = gevent.spawn(self._sender) - def __del__(self): - self.close() + def _send(self, parts): + e = None + for i in range(len(parts) - 1): + try: + self._socket.send(parts[i], copy=False, flags=zmq.SNDMORE) + except (gevent.GreenletExit, gevent.Timeout) as e: + if i == 0: + raise + self._socket.send(parts[i], copy=False, flags=zmq.SNDMORE) + try: + self._socket.send(parts[-1], copy=False) + except (gevent.GreenletExit, gevent.Timeout) as e: + self._socket.send(parts[-1], copy=False) + if e: + raise e + + def __call__(self, parts, timeout=None): + if timeout: + with gevent.Timeout(timeout): + self._send(parts) + else: + self._send(parts) + + +class SequentialReceiver(object): + + def __init__(self, socket): + self._socket = socket + + def _recv(self): + e = None + parts = [] + while True: + try: + part = self._socket.recv(copy=False) + except (gevent.GreenletExit, gevent.Timeout) as e: + if len(parts) == 0: + raise + part = self._socket.recv(copy=False) + parts.append(part) + if not part.more: + break + if e: + raise e + return parts + + def __call__(self, timeout=None): + if timeout: + with gevent.Timeout(timeout): + return self._recv() + else: + return self._recv() + + +class Sender(SequentialSender): + + def __init__(self, socket): + self._socket = socket + self._send_queue = gevent.queue.Channel() + self._send_task = gevent.spawn(self._sender) def close(self): if self._send_task: self._send_task.kill() def _sender(self): - running = True for parts in self._send_queue: - for i in xrange(len(parts) - 1): - try: - self._socket.send(parts[i], flags=zmq.SNDMORE) - except gevent.GreenletExit: - if i == 0: - return - running = False - self._socket.send(parts[i], flags=zmq.SNDMORE) - self._socket.send(parts[-1]) - if not running: - return + super(Sender, self)._send(parts) - def __call__(self, parts): - self._send_queue.put(parts) + def __call__(self, parts, timeout=None): + try: + self._send_queue.put(parts, timeout=timeout) + except gevent.queue.Full: + raise TimeoutExpired(timeout) -class Receiver(object): +class Receiver(SequentialReceiver): def __init__(self, socket): self._socket = socket - self._recv_queue = gevent.queue.Queue(maxsize=0) + self._recv_queue = gevent.queue.Channel() self._recv_task = gevent.spawn(self._recver) - def __del__(self): - self.close() - def close(self): if self._recv_task: self._recv_task.kill() + self._recv_queue = None def _recver(self): - running = True while True: - parts = [] - while True: - try: - part = self._socket.recv() - except gevent.GreenletExit: - running = False - if len(parts) == 0: - return - part = self._socket.recv() - parts.append(part) - if not self._socket.getsockopt(zmq.RCVMORE): - break - if not running: - break + parts = super(Receiver, self)._recv() self._recv_queue.put(parts) - def __call__(self): - return self._recv_queue.get() + def __call__(self, timeout=None): + try: + return self._recv_queue.get(timeout=timeout) + except gevent.queue.Empty: + raise TimeoutExpired(timeout) class Event(object): - __slots__ = [ '_name', '_args', '_header' ] + __slots__ = ['_name', '_args', '_header', '_identity'] + # protocol details: + # - `name` and `header` keys must be unicode strings. + # - `message_id` and 'response_to' values are opaque bytes string. + # - `v' value is an integer. def __init__(self, name, args, context, header=None): self._name = name self._args = args if header is None: - context = context - self._header = { - 'message_id': context.new_msgid(), - 'v': 3 - } + self._header = {u'message_id': context.new_msgid(), u'v': 3} else: self._header = header + self._identity = None @property def header(self): @@ -137,14 +195,30 @@ def name(self, v): def args(self): return self._args + @property + def identity(self): + return self._identity + + @identity.setter + def identity(self, v): + self._identity = v + def pack(self): - return msgpack.Packer().pack((self._header, self._name, self._args)) + payload = (self._header, self._name, self._args) + r = msgpack.Packer(use_bin_type=True).pack(payload) + return r @staticmethod def unpack(blob): - unpacker = msgpack.Unpacker() + unpacker = msgpack.Unpacker(raw=False) unpacker.feed(blob) - (header, name, args) = unpacker.unpack() + unpacked_msg = unpacker.unpack() + + try: + (header, name, args) = unpacked_msg + except Exception as e: + raise Exception('invalid msg format "{0}": {1}'.format( + unpacked_msg, e)) # Backward compatibility if not isinstance(header, dict): @@ -159,46 +233,78 @@ def __str__(self, ignore_args=False): args = self._args try: args = '<<{0}>>'.format(str(self.unpack(self._args))) - except: + except Exception: pass - return '{0} {1} {2}'.format(self._name, self._header, - args) + if self._identity: + identity = ', '.join(repr(x.bytes) for x in self._identity) + return '<{0}> {1} {2} {3}'.format(identity, self._name, + self._header, args) + return '{0} {1} {2}'.format(self._name, self._header, args) -class Events(object): +class Events(ChannelBase): def __init__(self, zmq_socket_type, context=None): + self._debug = False self._zmq_socket_type = zmq_socket_type self._context = context or Context.get_instance() - self._socket = zmq.Socket(self._context, zmq_socket_type) - self._send = self._socket.send_multipart - self._recv = self._socket.recv_multipart - if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.XREQ, zmq.XREP): + self._socket = self._context.socket(zmq_socket_type) + + if zmq_socket_type in (zmq.PUSH, zmq.PUB, zmq.DEALER, zmq.ROUTER): self._send = Sender(self._socket) - if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.XREQ, zmq.XREP): + elif zmq_socket_type in (zmq.REQ, zmq.REP): + self._send = SequentialSender(self._socket) + else: + self._send = None + + if zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.DEALER, zmq.ROUTER): self._recv = Receiver(self._socket) + elif zmq_socket_type in (zmq.REQ, zmq.REP): + self._recv = SequentialReceiver(self._socket) + else: + self._recv = None + + @property + def recv_is_supported(self): + return self._recv is not None @property - def recv_is_available(self): - return self._zmq_socket_type in (zmq.PULL, zmq.SUB, zmq.XREQ, zmq.XREP) + def emit_is_supported(self): + return self._send is not None def __del__(self): - if not self._socket.closed: - self.close() + try: + if not self._socket.closed: + self.close() + except (AttributeError, TypeError): + pass def close(self): try: self._send.close() - except AttributeError: + except (AttributeError, TypeError, gevent.GreenletExit): pass try: self._recv.close() - except AttributeError: + except (AttributeError, TypeError, gevent.GreenletExit): pass self._socket.close() + @property + def debug(self): + return self._debug + + @debug.setter + def debug(self, v): + if v != self._debug: + self._debug = v + if self._debug: + logger.debug('debug enabled') + else: + logger.debug('debug disabled') + def _resolve_endpoint(self, endpoint, resolve=True): if resolve: - endpoint = self._context.middleware_resolve_endpoint(endpoint) + endpoint = self._context.hook_resolve_endpoint(endpoint) if isinstance(endpoint, (tuple, list)): r = [] for sub_endpoint in endpoint: @@ -210,48 +316,56 @@ def connect(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.connect(endpoint_)) + logger.debug('connected to %s (status=%s)', endpoint_, r[-1]) return r def bind(self, endpoint, resolve=True): r = [] for endpoint_ in self._resolve_endpoint(endpoint, resolve): r.append(self._socket.bind(endpoint_)) + logger.debug('bound to %s (status=%s)', endpoint_, r[-1]) + return r + + def disconnect(self, endpoint, resolve=True): + r = [] + for endpoint_ in self._resolve_endpoint(endpoint, resolve): + r.append(self._socket.disconnect(endpoint_)) + logger.debug('disconnected from %s (status=%s)', endpoint_, r[-1]) return r - def create_event(self, name, args, xheader={}): + def new_event(self, name, args, xheader=None): event = Event(name, args, context=self._context) - for k, v in xheader.items(): - if k == 'zmqid': - continue - event.header[k] = v + if xheader: + event.header.update(xheader) return event - def emit_event(self, event, identity=None): - if identity is not None: - parts = list(identity) - parts.extend(['', event.pack()]) - elif self._zmq_socket_type in (zmq.XREQ, zmq.XREP): - parts = ('', event.pack()) + def emit_event(self, event, timeout=None): + if self._debug: + logger.debug('--> %s', event) + if event.identity: + parts = list(event.identity or list()) + parts.extend([b'', event.pack()]) + elif self._zmq_socket_type in (zmq.DEALER, zmq.ROUTER): + parts = (b'', event.pack()) else: parts = (event.pack(),) - self._send(parts) + self._send(parts, timeout) - def emit(self, name, args, xheader={}): - event = self.create_event(name, args, xheader) - identity = xheader.get('zmqid', None) - return self.emit_event(event, identity) - - def recv(self): - parts = self._recv() - if len(parts) == 1: - identity = None - blob = parts[0] - else: + def recv(self, timeout=None): + parts = self._recv(timeout=timeout) + if len(parts) > 2: identity = parts[0:-2] blob = parts[-1] - event = Event.unpack(blob) - if identity is not None: - event.header['zmqid'] = identity + elif len(parts) == 2: + identity = parts[0:-1] + blob = parts[-1] + else: + identity = None + blob = parts[0] + event = Event.unpack(get_pyzmq_frame_buffer(blob)) + event.identity = identity + if self._debug: + logger.debug('<-- %s', event) return event def setsockopt(self, *args): @@ -260,39 +374,3 @@ def setsockopt(self, *args): @property def context(self): return self._context - - -class WrappedEvents(object): - - def __init__(self, channel): - self._channel = channel - - def close(self): - pass - - @property - def recv_is_available(self): - return self._channel.recv_is_available - - def create_event(self, name, args, xheader={}): - event = Event(name, args, self._channel.context) - event.header.update(xheader) - return event - - def emit_event(self, event, identity=None): - event_payload = (event.header, event.name, event.args) - wrapper_event = self._channel.create_event('w', event_payload) - self._channel.emit_event(wrapper_event) - - def emit(self, name, args, xheader={}): - wrapper_event = self.create_event(name, args, xheader) - self.emit_event(wrapper_event) - - def recv(self, timeout=None): - wrapper_event = self._channel.recv() - (header, name, args) = wrapper_event.args - return Event(name, args, None, header) - - @property - def context(self): - return self._channel.context diff --git a/zerorpc/exceptions.py b/zerorpc/exceptions.py index 781200e..14a1419 100644 --- a/zerorpc/exceptions.py +++ b/zerorpc/exceptions.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/zerorpc/gevent_zmq.py b/zerorpc/gevent_zmq.py index fc751f1..54420ae 100644 --- a/zerorpc/gevent_zmq.py +++ b/zerorpc/gevent_zmq.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -25,13 +25,27 @@ # Based on https://github.com/traviscline/gevent-zeromq/ # We want to act like zmq -from zmq import * +from zmq import * # noqa + +try: + # Try to import enums for pyzmq >= 23.0.0 + from zmq.constants import * # noqa +except ImportError: + pass + + +# Explicit import to please flake8 +from zmq import ZMQError # A way to access original zmq import zmq as _zmq import gevent.event import gevent.core +import errno +from logging import getLogger + +logger = getLogger(__name__) class Context(_zmq.Context): @@ -47,17 +61,21 @@ class Socket(_zmq.Socket): def __init__(self, context, socket_type): super(Socket, self).__init__(context, socket_type) on_state_changed_fd = self.getsockopt(_zmq.FD) - self._readable = gevent.event.Event() - self._writable = gevent.event.Event() + # NOTE: pyzmq 13.0.0 messed up with setattr (they turned it into a + # non-op) and you can't assign attributes normally anymore, hence the + # tricks with self.__dict__ here + self.__dict__["_readable"] = gevent.event.Event() + self.__dict__["_writable"] = gevent.event.Event() try: # gevent>=1.0 - self._state_event = gevent.hub.get_hub().loop.io( + self.__dict__["_state_event"] = gevent.hub.get_hub().loop.io( on_state_changed_fd, gevent.core.READ) self._state_event.start(self._on_state_changed) except AttributeError: # gevent<1.0 - self._state_event = gevent.core.read_event(on_state_changed_fd, - self._on_state_changed, persist=True) + self.__dict__["_state_event"] = \ + gevent.core.read_event(on_state_changed_fd, + self._on_state_changed, persist=True) def _on_state_changed(self, event=None, _evtype=None): if self.closed: @@ -65,7 +83,14 @@ def _on_state_changed(self, event=None, _evtype=None): self._readable.set() return - events = self.getsockopt(_zmq.EVENTS) + while True: + try: + events = self.getsockopt(_zmq.EVENTS) + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise + if events & _zmq.POLLOUT: self._writable.set() if events & _zmq.POLLIN: @@ -81,18 +106,53 @@ def close(self): self._state_event.cancel() super(Socket, self).close() + def connect(self, *args, **kwargs): + while True: + try: + return super(Socket, self).connect(*args, **kwargs) + except _zmq.ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise + def send(self, data, flags=0, copy=True, track=False): if flags & _zmq.NOBLOCK: return super(Socket, self).send(data, flags, copy, track) flags |= _zmq.NOBLOCK while True: try: - return super(Socket, self).send(data, flags, copy, track) - except _zmq.ZMQError, e: - if e.errno != _zmq.EAGAIN: + msg = super(Socket, self).send(data, flags, copy, track) + # The following call, force polling the state of the zmq socket + # (POLLIN and/or POLLOUT). It seems that a POLLIN event is often + # missed when the socket is used to send at the same time, + # forcing to poll at this exact moment seems to reduce the + # latencies when a POLLIN event is missed. The drawback is a + # reduced throughput (roughly 8.3%) in exchange of a normal + # concurrency. In other hand, without the following line, you + # loose 90% of the performances as soon as there is simultaneous + # send and recv on the socket. + self._on_state_changed() + return msg + except _zmq.ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._writable.clear() - self._writable.wait() + # The following sleep(0) force gevent to switch out to another + # coroutine and seems to refresh the notion of time that gevent may + # have. This definitively eliminate the gevent bug that can trigger + # a timeout too soon under heavy load. In theory it will incur more + # CPU usage, but in practice it balance even with the extra CPU used + # when the timeout triggers too soon in the following loop. So for + # the same CPU load, you get a better throughput (roughly 18.75%). + gevent.sleep(0) + while not self._writable.wait(timeout=1): + try: + if self.getsockopt(_zmq.EVENTS) & _zmq.POLLOUT: + logger.error("/!\\ gevent_zeromq BUG /!\\ " + "catching up after missing event (SEND) /!\\") + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise def recv(self, flags=0, copy=True, track=False): if flags & _zmq.NOBLOCK: @@ -100,14 +160,36 @@ def recv(self, flags=0, copy=True, track=False): flags |= _zmq.NOBLOCK while True: try: - return super(Socket, self).recv(flags, copy, track) - except _zmq.ZMQError, e: - if e.errno != _zmq.EAGAIN: + msg = super(Socket, self).recv(flags, copy, track) + # The following call, force polling the state of the zmq socket + # (POLLIN and/or POLLOUT). It seems that a POLLOUT event is + # often missed when the socket is used to receive at the same + # time, forcing to poll at this exact moment seems to reduce the + # latencies when a POLLOUT event is missed. The drawback is a + # reduced throughput (roughly 8.3%) in exchange of a normal + # concurrency. In other hand, without the following line, you + # loose 90% of the performances as soon as there is simultaneous + # send and recv on the socket. + self._on_state_changed() + return msg + except _zmq.ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): raise self._readable.clear() - while not self._readable.wait(timeout=0.5): - events = self.getsockopt(_zmq.EVENTS) - if bool(events & _zmq.POLLIN): - print "/!\\ gevent_zeromq BUG /!\\ " \ - "catching after missing event /!\\" - break + # The following sleep(0) force gevent to switch out to another + # coroutine and seems to refresh the notion of time that gevent may + # have. This definitively eliminate the gevent bug that can trigger + # a timeout too soon under heavy load. In theory it will incur more + # CPU usage, but in practice it balance even with the extra CPU used + # when the timeout triggers too soon in the following loop. So for + # the same CPU load, you get a better throughput (roughly 18.75%). + gevent.sleep(0) + while not self._readable.wait(timeout=1): + try: + if self.getsockopt(_zmq.EVENTS) & _zmq.POLLIN: + logger.error("/!\\ gevent_zeromq BUG /!\\ " + "catching up after missing event (RECV) /!\\") + break + except ZMQError as e: + if e.errno not in (_zmq.EAGAIN, errno.EINTR): + raise diff --git a/zerorpc/heartbeat.py b/zerorpc/heartbeat.py index e7af22f..23b974d 100644 --- a/zerorpc/heartbeat.py +++ b/zerorpc/heartbeat.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -28,17 +28,19 @@ import gevent.queue import gevent.event import gevent.local -import gevent.coros +import gevent.lock -from .exceptions import * +from .exceptions import LostRemote, TimeoutExpired +from .channel_base import ChannelBase -class HeartBeatOnChannel(object): +class HeartBeatOnChannel(ChannelBase): def __init__(self, channel, freq=5, passive=False): + self._closed = False self._channel = channel self._heartbeat_freq = freq - self._input_queue = gevent.queue.Queue(maxsize=0) + self._input_queue = gevent.queue.Channel() self._remote_last_hb = None self._lost_remote = False self._recv_task = gevent.spawn(self._recver) @@ -49,13 +51,15 @@ def __init__(self, channel, freq=5, passive=False): self._start_heartbeat() @property - def recv_is_available(self): - return self._channel.recv_is_available + def recv_is_supported(self): + return self._channel.recv_is_supported - def __del__(self): - self.close() + @property + def emit_is_supported(self): + return self._channel.emit_is_supported def close(self): + self._closed = True if self._heartbeat_task is not None: self._heartbeat_task.kill() self._heartbeat_task = None @@ -73,25 +77,26 @@ def _heartbeat(self): self._remote_last_hb = time.time() if time.time() > self._remote_last_hb + self._heartbeat_freq * 2: self._lost_remote = True - gevent.kill(self._parent_coroutine, - self._lost_remote_exception()) + if not self._closed: + gevent.kill(self._parent_coroutine, + self._lost_remote_exception()) break - self._channel.emit('_zpc_hb', (0,)) # 0 -> compat with protocol v2 + self._channel.emit(u'_zpc_hb', (0,)) # 0 -> compat with protocol v2 def _start_heartbeat(self): - if self._heartbeat_task is None and self._heartbeat_freq is not None: + if self._heartbeat_task is None and self._heartbeat_freq is not None and not self._closed: self._heartbeat_task = gevent.spawn(self._heartbeat) def _recver(self): while True: event = self._channel.recv() if self._compat_v2 is None: - self._compat_v2 = event.header.get('v', 0) < 3 - if event.name == '_zpc_hb': + self._compat_v2 = event.header.get(u'v', 0) < 3 + if event.name == u'_zpc_hb': self._remote_last_hb = time.time() self._start_heartbeat() if self._compat_v2: - event.name = '_zpc_more' + event.name = u'_zpc_more' self._input_queue.put(event) else: self._input_queue.put(event) @@ -100,31 +105,24 @@ def _lost_remote_exception(self): return LostRemote('Lost remote after {0}s heartbeat'.format( self._heartbeat_freq * 2)) - def create_event(self, name, args, xheader={}): - if self._compat_v2 and name == '_zpc_more': - name = '_zpc_hb' - return self._channel.create_event(name, args, xheader) + def new_event(self, name, args, header=None): + if self._compat_v2 and name == u'_zpc_more': + name = u'_zpc_hb' + return self._channel.new_event(name, args, header) - def emit_event(self, event): + def emit_event(self, event, timeout=None): if self._lost_remote: raise self._lost_remote_exception() - self._channel.emit_event(event) - - def emit(self, name, args, xheader={}): - event = self.create_event(name, args, xheader) - self.emit_event(event) + self._channel.emit_event(event, timeout) def recv(self, timeout=None): if self._lost_remote: raise self._lost_remote_exception() - try: - event = self._input_queue.get(timeout=timeout) + return self._input_queue.get(timeout=timeout) except gevent.queue.Empty: raise TimeoutExpired(timeout) - return event - @property def channel(self): return self._channel diff --git a/zerorpc/patterns.py b/zerorpc/patterns.py index 08da7cf..3623e17 100644 --- a/zerorpc/patterns.py +++ b/zerorpc/patterns.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,53 +23,73 @@ # SOFTWARE. +class ReqRep(object): -class ReqRep: - - def process_call(self, context, bufchan, event, functor): - result = context.middleware_call_procedure(functor, *event.args) - bufchan.emit('OK', (result,), context.middleware_get_task_context()) + def process_call(self, context, channel, req_event, functor): + context.hook_server_before_exec(req_event) + result = functor(*req_event.args) + rep_event = channel.new_event(u'OK', (result,), + context.hook_get_task_context()) + context.hook_server_after_exec(req_event, rep_event) + channel.emit_event(rep_event) def accept_answer(self, event): - return True - - def process_answer(self, context, bufchan, event, method, - raise_remote_error): - result = event.args[0] - if event.name == 'ERR': - raise_remote_error(event) - bufchan.close() - return result - - -class ReqStream: - - def process_call(self, context, bufchan, event, functor): - xheader = context.middleware_get_task_context() - for result in iter(context.middleware_call_procedure(functor, - *event.args)): - bufchan.emit('STREAM', result, xheader) - bufchan.emit('STREAM_DONE', None, xheader) + return event.name in (u'OK', u'ERR') + + def process_answer(self, context, channel, req_event, rep_event, + handle_remote_error): + try: + if rep_event.name == u'ERR': + exception = handle_remote_error(rep_event) + context.hook_client_after_request(req_event, rep_event, exception) + raise exception + context.hook_client_after_request(req_event, rep_event) + return rep_event.args[0] + finally: + channel.close() + + +class ReqStream(object): + + def process_call(self, context, channel, req_event, functor): + context.hook_server_before_exec(req_event) + xheader = context.hook_get_task_context() + for result in iter(functor(*req_event.args)): + channel.emit(u'STREAM', result, xheader) + done_event = channel.new_event(u'STREAM_DONE', None, xheader) + # NOTE: "We" made the choice to call the hook once the stream is done, + # the other choice was to call it at each iteration. I donu't think that + # one choice is better than the other, so Iu'm fine with changing this + # or adding the server_after_iteration and client_after_iteration hooks. + context.hook_server_after_exec(req_event, done_event) + channel.emit_event(done_event) def accept_answer(self, event): - return event.name in ('STREAM', 'STREAM_DONE') - - def process_answer(self, context, bufchan, event, method, - raise_remote_error): - - def is_stream_done(event): - return event.name == 'STREAM_DONE' - bufchan.on_close_if = is_stream_done - - def iterator(event): - while event.name == 'STREAM': - yield event.args - event = bufchan.recv() - if event.name == 'ERR': - raise_remote_error(event) - bufchan.close() - - return iterator(event) + return event.name in (u'STREAM', u'STREAM_DONE') + + def process_answer(self, context, channel, req_event, rep_event, + handle_remote_error): + + def is_stream_done(rep_event): + return rep_event.name == u'STREAM_DONE' + channel.on_close_if = is_stream_done + + def iterator(req_event, rep_event): + try: + while rep_event.name == u'STREAM': + # Like in process_call, we made the choice to call the + # after_exec hook only when the stream is done. + yield rep_event.args + rep_event = channel.recv() + if rep_event.name == u'ERR': + exception = handle_remote_error(rep_event) + context.hook_client_after_request(req_event, rep_event, exception) + raise exception + context.hook_client_after_request(req_event, rep_event) + finally: + channel.close() + + return iterator(req_event, rep_event) patterns_list = [ReqStream(), ReqRep()] diff --git a/zerorpc/socket.py b/zerorpc/socket.py index 2a7020e..35cb7e4 100644 --- a/zerorpc/socket.py +++ b/zerorpc/socket.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -41,3 +41,14 @@ def connect(self, endpoint, resolve=True): def bind(self, endpoint, resolve=True): return self._events.bind(endpoint, resolve) + + def disconnect(self, endpoint, resolve=True): + return self._events.disconnect(endpoint, resolve) + + @property + def debug(self): + return self._events.debug + + @debug.setter + def debug(self, v): + self._events.debug = v diff --git a/zerorpc/version.py b/zerorpc/version.py index 9d90dab..4324a5a 100644 --- a/zerorpc/version.py +++ b/zerorpc/version.py @@ -2,7 +2,7 @@ # Open Source Initiative OSI - The MIT License (MIT):Licensing # # The MIT License (MIT) -# Copyright (c) 2012 DotCloud Inc (opensource@dotcloud.com) +# Copyright (c) 2015 François-Xavier Bourlet (bombela+zerorpc@gmail.com) # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -23,7 +23,7 @@ # SOFTWARE. __title__ = 'zerorpc' -__version__ = '0.2.1' -__author__ = 'dotCloud, Inc.' +__version__ = '0.6.3' +__author__ = 'François-Xavier Bourlet .' __license__ = 'MIT' -__copyright__ = 'Copyright 2012 dotCloud, Inc.' +__copyright__ = 'Copyright 2015 François-Xavier Bourlet .'