From 93ad88a9a8fe2ea8d96fb1d2a0f1625a3c5fee7c Mon Sep 17 00:00:00 2001 From: Alex Coplan Date: Mon, 4 Nov 2019 11:54:49 +0000 Subject: [PATCH 001/104] fix type hints on client/server args * Make ping_interval et al. optional so that code that passes None here will type check. --- src/websockets/client.py | 8 ++++---- src/websockets/server.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/websockets/client.py b/src/websockets/client.py index eb58f9f48..831b70805 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -373,11 +373,11 @@ def __init__( *, path: Optional[str] = None, create_protocol: Optional[Type[WebSocketClientProtocol]] = None, - ping_interval: float = 20, - ping_timeout: float = 20, + ping_interval: Optional[float] = 20, + ping_timeout: Optional[float] = 20, close_timeout: Optional[float] = None, - max_size: int = 2 ** 20, - max_queue: int = 2 ** 5, + max_size: Optional[int] = 2 ** 20, + max_queue: Optional[int] = 2 ** 5, read_limit: int = 2 ** 16, write_limit: int = 2 ** 16, loop: Optional[asyncio.AbstractEventLoop] = None, diff --git a/src/websockets/server.py b/src/websockets/server.py index 4f5e9e0ef..0313fa848 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -842,11 +842,11 @@ def __init__( *, path: Optional[str] = None, create_protocol: Optional[Type[WebSocketServerProtocol]] = None, - ping_interval: float = 20, - ping_timeout: float = 20, + ping_interval: Optional[float] = 20, + ping_timeout: Optional[float] = 20, close_timeout: Optional[float] = None, - max_size: int = 2 ** 20, - max_queue: int = 2 ** 5, + max_size: Optional[int] = 2 ** 20, + max_queue: Optional[int] = 2 ** 5, read_limit: int = 2 ** 16, write_limit: int = 2 ** 16, loop: Optional[asyncio.AbstractEventLoop] = None, From 3bab7fd155636c73b79b258de752b36687bba347 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 16 Nov 2019 20:37:14 +0100 Subject: [PATCH 002/104] Clarify local/remote_address after connection is closed. Fix #688. --- src/websockets/protocol.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 6c29b2a52..e065bef67 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -357,10 +357,9 @@ def secure(self) -> Optional[bool]: @property def local_address(self) -> Any: """ - Local address of the connection. + Local address of the connection as a ``(host, port)`` tuple. - This is a ``(host, port)`` tuple or ``None`` if the connection hasn't - been established yet. + When the connection isn't open, ``local_address`` is ``None``. """ try: @@ -373,10 +372,9 @@ def local_address(self) -> Any: @property def remote_address(self) -> Any: """ - Remote address of the connection. + Remote address of the connection as a ``(host, port)`` tuple. - This is a ``(host, port)`` tuple or ``None`` if the connection hasn't - been established yet. + When the connection isn't open, ``remote_address`` is ``None``. """ try: From 910f417c9179150c5ab4b44c7361dbf1e51ec322 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 16 Nov 2019 20:40:15 +0100 Subject: [PATCH 003/104] Always reraise CancelledError. It's really hard to write tests for this :-( Fix #672. --- src/websockets/client.py | 2 ++ src/websockets/server.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/websockets/client.py b/src/websockets/client.py index 831b70805..f92350249 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -99,6 +99,8 @@ async def read_http_response(self) -> Tuple[int, Headers]: """ try: status_code, reason, headers = await read_response(self.reader) + except asyncio.CancelledError: # pragma: no cover + raise except Exception as exc: raise InvalidMessage("did not receive a valid HTTP response") from exc diff --git a/src/websockets/server.py b/src/websockets/server.py index 0313fa848..f872262ef 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -133,6 +133,8 @@ async def handler(self) -> None: available_subprotocols=self.available_subprotocols, extra_headers=self.extra_headers, ) + except asyncio.CancelledError: # pragma: no cover + raise except ConnectionError: logger.debug("Connection error in opening handshake", exc_info=True) raise @@ -231,6 +233,8 @@ async def read_http_request(self) -> Tuple[str, Headers]: """ try: path, headers = await read_request(self.reader) + except asyncio.CancelledError: # pragma: no cover + raise except Exception as exc: raise InvalidMessage("did not receive a valid HTTP request") from exc From a1615b47fcd416e5016d7e471976314c267f4349 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 14 Jan 2020 20:53:41 +0200 Subject: [PATCH 004/104] Fix for Python 3.10: use sys.version_info instead of sys.version --- src/websockets/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/websockets/http.py b/src/websockets/http.py index ba6d274bf..f87bfb76a 100644 --- a/src/websockets/http.py +++ b/src/websockets/http.py @@ -36,7 +36,8 @@ MAX_HEADERS = 256 MAX_LINE = 4096 -USER_AGENT = f"Python/{sys.version[:3]} websockets/{websockets_version}" +PYTHON_VERSION = "{}.{}".format(*sys.version_info) +USER_AGENT = f"Python/{PYTHON_VERSION} websockets/{websockets_version}" def d(value: bytes) -> str: From 160dfbec7dd582c12817de5c85e6bf3fbbc34826 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 25 Jan 2020 21:10:09 +0100 Subject: [PATCH 005/104] Clarify comment about RFC inconsistency. --- src/websockets/handshake.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/websockets/handshake.py b/src/websockets/handshake.py index 9bfe27754..646b6dba4 100644 --- a/src/websockets/handshake.py +++ b/src/websockets/handshake.py @@ -87,7 +87,8 @@ def check_request(headers: Headers) -> str: ) # For compatibility with non-strict implementations, ignore case when - # checking the Upgrade header. It's supposed to be 'WebSocket'. + # checking the Upgrade header. The RFC always uses "websocket", except + # in section 11.2. (IANA registration) where it uses "WebSocket". if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): raise InvalidUpgrade("Upgrade", ", ".join(upgrade)) @@ -163,7 +164,8 @@ def check_response(headers: Headers, key: str) -> None: ) # For compatibility with non-strict implementations, ignore case when - # checking the Upgrade header. It's supposed to be 'WebSocket'. + # checking the Upgrade header. The RFC always uses "websocket", except + # in section 11.2. (IANA registration) where it uses "WebSocket". if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): raise InvalidUpgrade("Upgrade", ", ".join(upgrade)) From 4f1964295ad0e81c8c96b99c3fe9dafc96f11f28 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 18 Feb 2020 22:09:55 +0100 Subject: [PATCH 006/104] Speculation about proof-of-stake gets old. Meanwhile, bitcoin still heats the planet. Sorry crypto buffs. Refs #480 and several others. --- docs/contributing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 40f1dbb54..61c0b979c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -53,6 +53,9 @@ Bitcoin users websockets appears to be quite popular for interfacing with Bitcoin or other cryptocurrency trackers. I'm strongly opposed to Bitcoin's carbon footprint. +I'm aware of efforts to build proof-of-stake models. I'll care once the total +carbon footprint of all cryptocurrencies drops to a non-bullshit level. + Please stop heating the planet where my children are supposed to live, thanks. Since ``websockets`` is released under an open-source license, you can use it From 6b5cbaf41cdbc9a2074e357ccc613ef25517dd32 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 1 Mar 2020 19:10:49 +1100 Subject: [PATCH 007/104] Fix simple typo: severel -> several There is a small typo in src/websockets/client.py, src/websockets/server.py. Should read `several` rather than `severel`. --- src/websockets/client.py | 2 +- src/websockets/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/websockets/client.py b/src/websockets/client.py index f92350249..be055310d 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -134,7 +134,7 @@ def process_extensions( client configuration. If no match is found, an exception is raised. If several variants of the same extension are accepted by the server, - it may be configured severel times, which won't make sense in general. + it may be configured several times, which won't make sense in general. Extensions must implement their own requirements. For this purpose, the list of previously accepted extensions is provided. diff --git a/src/websockets/server.py b/src/websockets/server.py index f872262ef..1d8de8914 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -369,7 +369,7 @@ def process_extensions( server configuration. If no match is found, the extension is ignored. If several variants of the same extension are proposed by the client, - it may be accepted severel times, which won't make sense in general. + it may be accepted several times, which won't make sense in general. Extensions must implement their own requirements. For this purpose, the list of previously accepted extensions is provided. From 18dbc49c935285e35a54e46030d326f3a49ea7b7 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 23 May 2020 07:46:06 +0200 Subject: [PATCH 008/104] Run tests against the latest Python 3.8. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0877c161a..68d02416d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,7 +31,7 @@ jobs: - run: tox -e py37 py38: docker: - - image: circleci/python:3.8.0rc1 + - image: circleci/python:3.8 steps: # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc From 6170e235723f27a5aaa42ea86828f0266cc004f9 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 23 May 2020 09:30:16 +0200 Subject: [PATCH 009/104] Don't attempt to build wheels on PyPy 2.7. --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 7954ee4be..2db489a76 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,7 +6,7 @@ skip_branch_with_pr: true environment: # websockets only works on Python >= 3.6. - CIBW_SKIP: cp27-* cp33-* cp34-* cp35-* + CIBW_SKIP: cp27-* cp33-* cp34-* cp35-* pp27-* CIBW_TEST_COMMAND: python -W default -m unittest WEBSOCKETS_TESTS_TIMEOUT_FACTOR: 100 From 46e8fb5cecb474991e18f7b809378b7d76477df2 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 23 May 2020 09:58:58 +0200 Subject: [PATCH 010/104] Fix flake8 violation. --- src/websockets/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index e065bef67..2082c81fc 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -349,7 +349,7 @@ def port(self) -> Optional[int]: @property def secure(self) -> Optional[bool]: - warnings.warn(f"don't use secure", DeprecationWarning) + warnings.warn("don't use secure", DeprecationWarning) return self._secure # Public API From 68dfb14963ea12e0068aefbbb43f101113d0750d Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 23 May 2020 10:15:51 +0200 Subject: [PATCH 011/104] Don't attempt to build wheels on PyPy 2.7 (bis). --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 030693759..6234bb649 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ env: global: # websockets only works on Python >= 3.6. - - CIBW_SKIP="cp27-* cp33-* cp34-* cp35-*" + - CIBW_SKIP="cp27-* cp33-* cp34-* cp35-* pp27-*" - CIBW_TEST_COMMAND="python3 -W default -m unittest" - WEBSOCKETS_TESTS_TIMEOUT_FACTOR=100 From fafcf65d430149a8b94379f9557655828a0dcdab Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 23 May 2020 12:56:22 +0200 Subject: [PATCH 012/104] Only build wheels on supported CPython versions. PyPy 3 wheels were failing to build on macOS. --- .appveyor.yml | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 2db489a76..d34b15aed 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,7 +6,7 @@ skip_branch_with_pr: true environment: # websockets only works on Python >= 3.6. - CIBW_SKIP: cp27-* cp33-* cp34-* cp35-* pp27-* + CIBW_BUILD: cp36-* cp37-* cp38-* CIBW_TEST_COMMAND: python -W default -m unittest WEBSOCKETS_TESTS_TIMEOUT_FACTOR: 100 diff --git a/.travis.yml b/.travis.yml index 6234bb649..26e1de60e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ env: global: # websockets only works on Python >= 3.6. - - CIBW_SKIP="cp27-* cp33-* cp34-* cp35-* pp27-*" + - CIBW_BUILD="cp36-* cp37-* cp38-*" - CIBW_TEST_COMMAND="python3 -W default -m unittest" - WEBSOCKETS_TESTS_TIMEOUT_FACTOR=100 From 69c94af5c0ad19402e0bedcc6b61a23fa070c946 Mon Sep 17 00:00:00 2001 From: David Bordeynik Date: Mon, 18 May 2020 10:38:08 +0300 Subject: [PATCH 013/104] Future-proof asyncio.wait usage. Fix #762. --- .circleci/config.yml | 12 ++++++++++++ .gitignore | 1 + src/websockets/server.py | 5 ++++- tox.ini | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 68d02416d..7be85d7f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,6 +38,15 @@ jobs: - checkout - run: sudo pip install tox - run: tox -e py38 + py39: + docker: + - image: circleci/python:3.9.0b1 + steps: + # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. + - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc + - checkout + - run: sudo pip install tox + - run: tox -e py39 workflows: version: 2 @@ -53,3 +62,6 @@ workflows: - py38: requires: - main + - py39: + requires: + - main diff --git a/.gitignore b/.gitignore index ef0d16520..c23cf5210 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc *.so .coverage +.idea/ .mypy_cache .tox build/ diff --git a/src/websockets/server.py b/src/websockets/server.py index 1d8de8914..e9318a4df 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -714,7 +714,10 @@ async def _close(self) -> None: # asyncio.wait doesn't accept an empty first argument if self.websockets: await asyncio.wait( - [websocket.close(1001) for websocket in self.websockets], + [ + asyncio.ensure_future(websocket.close(1001)) + for websocket in self.websockets + ], loop=self.loop if sys.version_info[:2] < (3, 8) else None, ) diff --git a/tox.ini b/tox.ini index 825e34061..cc224f9c6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37,py38,coverage,black,flake8,isort,mypy +envlist = py36,py37,py38,py39,coverage,black,flake8,isort,mypy [testenv] commands = python -W default -m unittest {posargs} From 24a77def7097cb7ae651edf35582c8def5a6ad3e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 13 Jun 2020 17:41:06 +0200 Subject: [PATCH 014/104] Update to mypy 0.780. --- src/websockets/__main__.py | 8 ++++---- src/websockets/typing.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/websockets/__main__.py b/src/websockets/__main__.py index 394f7ac79..1a720498d 100644 --- a/src/websockets/__main__.py +++ b/src/websockets/__main__.py @@ -49,10 +49,10 @@ def exit_from_event_loop_thread( if not stop.done(): # When exiting the thread that runs the event loop, raise # KeyboardInterrupt in the main thread to exit the program. - try: - ctrl_c = signal.CTRL_C_EVENT # Windows - except AttributeError: - ctrl_c = signal.SIGINT # POSIX + if sys.platform == "win32": + ctrl_c = signal.CTRL_C_EVENT + else: + ctrl_c = signal.SIGINT os.kill(os.getpid(), ctrl_c) diff --git a/src/websockets/typing.py b/src/websockets/typing.py index 4a60f93f6..a5062bc4b 100644 --- a/src/websockets/typing.py +++ b/src/websockets/typing.py @@ -14,7 +14,7 @@ """ # Remove try / except when dropping support for Python < 3.7 try: - Data.__doc__ = Data__doc__ # type: ignore + Data.__doc__ = Data__doc__ except AttributeError: # pragma: no cover pass @@ -31,7 +31,7 @@ ExtensionParameter__doc__ = """Parameter of a WebSocket extension""" try: - ExtensionParameter.__doc__ = ExtensionParameter__doc__ # type: ignore + ExtensionParameter.__doc__ = ExtensionParameter__doc__ except AttributeError: # pragma: no cover pass @@ -40,7 +40,7 @@ ExtensionHeader__doc__ = """Item parsed in a Sec-WebSocket-Extensions header""" try: - ExtensionHeader.__doc__ = ExtensionHeader__doc__ # type: ignore + ExtensionHeader.__doc__ = ExtensionHeader__doc__ except AttributeError: # pragma: no cover pass From 017a072705408d3df945e333e5edd93e0aa8c706 Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Fri, 12 Jun 2020 23:16:57 +0300 Subject: [PATCH 015/104] Fix exception causes in server.py --- src/websockets/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/websockets/server.py b/src/websockets/server.py index e9318a4df..0f0b51a7c 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -340,8 +340,8 @@ def process_origin( # per https://tools.ietf.org/html/rfc6454#section-7.3. try: origin = cast(Origin, headers.get("Origin")) - except MultipleValuesError: - raise InvalidHeader("Origin", "more than one Origin header found") + except MultipleValuesError as exc: + raise InvalidHeader("Origin", "more than one Origin header found") from exc if origins is not None: if origin not in origins: raise InvalidOrigin(origin) From 17499930cec591778d13e594b0cb978a9961e276 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 13:39:43 +0200 Subject: [PATCH 016/104] Ignore coverage measurement issue. --- src/websockets/protocol.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 2082c81fc..803970205 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -1175,7 +1175,9 @@ async def close_connection(self) -> None: # A client should wait for a TCP close from the server. if self.is_client and hasattr(self, "transfer_data_task"): if await self.wait_for_connection_lost(): - return + # Coverage marks this line as a partially executed branch. + # I supect a bug in coverage. Ignore it for now. + return # pragma: no cover logger.debug("%s ! timed out waiting for TCP close", self.side) # Half-close the TCP connection if possible (when there's no TLS). @@ -1184,7 +1186,9 @@ async def close_connection(self) -> None: self.transport.write_eof() if await self.wait_for_connection_lost(): - return + # Coverage marks this line as a partially executed branch. + # I supect a bug in coverage. Ignore it for now. + return # pragma: no cover logger.debug("%s ! timed out waiting for TCP close", self.side) finally: @@ -1210,7 +1214,9 @@ async def close_connection(self) -> None: self.transport.abort() # connection_lost() is called quickly after aborting. - await self.wait_for_connection_lost() + # Coverage marks this line as a partially executed branch. + # I supect a bug in coverage. Ignore it for now. + await self.wait_for_connection_lost() # pragma: no cover async def wait_for_connection_lost(self) -> bool: """ From f0cfa6ba2abf6d4b032b30cfae9d321e583d546e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 18:04:08 +0200 Subject: [PATCH 017/104] Realign docstring with Python version. --- src/websockets/speedups.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/websockets/speedups.c b/src/websockets/speedups.c index d1c2b37e6..ede181e5d 100644 --- a/src/websockets/speedups.c +++ b/src/websockets/speedups.c @@ -181,7 +181,7 @@ static PyMethodDef speedups_methods[] = { "apply_mask", (PyCFunction)apply_mask, METH_VARARGS | METH_KEYWORDS, - "Apply masking to websocket message.", + "Apply masking to the data of a WebSocket message.", }, {NULL, NULL, 0, NULL}, /* Sentinel */ }; From daad5180e09af5d860edf4191fb1791eb6b57cc8 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 20:21:51 +0200 Subject: [PATCH 018/104] =?UTF-8?q?Upgrade=20to=20isort=20=E2=89=A5=205.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- setup.cfg | 6 +----- tox.ini | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index d9e16fefe..06832945c 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export PYTHONPATH=src default: coverage style style: - isort --recursive src tests + isort src tests black src tests flake8 src tests mypy --strict src diff --git a/setup.cfg b/setup.cfg index c306b2d4f..02e70cdf5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,13 +9,9 @@ ignore = E731,F403,F405,W503 max-line-length = 88 [isort] +profile = black combine_as_imports = True -force_grid_wrap = 0 -include_trailing_comma = True -known_standard_library = asyncio -line_length = 88 lines_after_imports = 2 -multi_line_output = 3 [coverage:run] branch = True diff --git a/tox.ini b/tox.ini index cc224f9c6..b5488e5b0 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ commands = flake8 src tests deps = flake8 [testenv:isort] -commands = isort --check-only --recursive src tests +commands = isort --check-only src tests deps = isort [testenv:mypy] From 85b3fd67490bc1e5aa9e46c292c00aceeaa0d40b Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 6 Oct 2019 10:28:03 +0200 Subject: [PATCH 019/104] Move Headers class to its own module. This allows breaking an import loop. --- docs/api.rst | 6 ++ docs/changelog.rst | 12 ++- src/websockets/__init__.py | 1 + src/websockets/auth.py | 2 +- src/websockets/client.py | 3 +- src/websockets/datastructures.py | 159 ++++++++++++++++++++++++++++ src/websockets/exceptions.py | 2 +- src/websockets/handshake.py | 2 +- src/websockets/http.py | 173 ++----------------------------- src/websockets/protocol.py | 2 +- src/websockets/server.py | 3 +- tests/test_client_server.py | 3 +- tests/test_datastructures.py | 131 +++++++++++++++++++++++ tests/test_exceptions.py | 2 +- tests/test_handshake.py | 2 +- tests/test_http.py | 114 -------------------- 16 files changed, 330 insertions(+), 287 deletions(-) create mode 100644 src/websockets/datastructures.py create mode 100644 tests/test_datastructures.py diff --git a/docs/api.rst b/docs/api.rst index d265a91c2..f7706ee2c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -115,6 +115,12 @@ HTTP Basic Auth .. automethod:: process_request +Data structures +............... + +.. automodule:: websockets.datastructures + :members: + Exceptions .......... diff --git a/docs/changelog.rst b/docs/changelog.rst index 04f18a765..5de7357ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,11 +3,21 @@ Changelog .. currentmodule:: websockets -8.2 +9.0 ... *In development* +.. note:: + + **Version 9.0 moves or deprecates several low-level APIs.** + + * Import :class:`~datastructures.Headers` and + :exc:`~datastructures.MultipleValuesError` from + :mod:`websockets.datastructures` instead of :mod:`websockets.http`. + + Aliases provide backwards compatibility for all previously public APIs. + 8.1 ... diff --git a/src/websockets/__init__.py b/src/websockets/__init__.py index ea1d829a3..89829235c 100644 --- a/src/websockets/__init__.py +++ b/src/websockets/__init__.py @@ -2,6 +2,7 @@ from .auth import * # noqa from .client import * # noqa +from .datastructures import * # noqa from .exceptions import * # noqa from .protocol import * # noqa from .server import * # noqa diff --git a/src/websockets/auth.py b/src/websockets/auth.py index ae204b8d9..8198cd9d0 100644 --- a/src/websockets/auth.py +++ b/src/websockets/auth.py @@ -9,9 +9,9 @@ import http from typing import Any, Awaitable, Callable, Iterable, Optional, Tuple, Type, Union +from .datastructures import Headers from .exceptions import InvalidHeader from .headers import build_www_authenticate_basic, parse_authorization_basic -from .http import Headers from .server import HTTPResponse, WebSocketServerProtocol diff --git a/src/websockets/client.py b/src/websockets/client.py index be055310d..26a369c47 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -11,6 +11,7 @@ from types import TracebackType from typing import Any, Generator, List, Optional, Sequence, Tuple, Type, cast +from .datastructures import Headers, HeadersLike from .exceptions import ( InvalidHandshake, InvalidHeader, @@ -30,7 +31,7 @@ parse_extension, parse_subprotocol, ) -from .http import USER_AGENT, Headers, HeadersLike, read_response +from .http import USER_AGENT, read_response from .protocol import WebSocketCommonProtocol from .typing import ExtensionHeader, Origin, Subprotocol from .uri import WebSocketURI, parse_uri diff --git a/src/websockets/datastructures.py b/src/websockets/datastructures.py new file mode 100644 index 000000000..f70d92ad7 --- /dev/null +++ b/src/websockets/datastructures.py @@ -0,0 +1,159 @@ +""" +This module defines a data structure for manipulating HTTP headers. + +""" + +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + MutableMapping, + Tuple, + Union, +) + + +__all__ = ["Headers", "MultipleValuesError"] + + +class MultipleValuesError(LookupError): + """ + Exception raised when :class:`Headers` has more than one value for a key. + + """ + + def __str__(self) -> str: + # Implement the same logic as KeyError_str in Objects/exceptions.c. + if len(self.args) == 1: + return repr(self.args[0]) + return super().__str__() + + +class Headers(MutableMapping[str, str]): + """ + Efficient data structure for manipulating HTTP headers. + + A :class:`list` of ``(name, values)`` is inefficient for lookups. + + A :class:`dict` doesn't suffice because header names are case-insensitive + and multiple occurrences of headers with the same name are possible. + + :class:`Headers` stores HTTP headers in a hybrid data structure to provide + efficient insertions and lookups while preserving the original data. + + In order to account for multiple values with minimal hassle, + :class:`Headers` follows this logic: + + - When getting a header with ``headers[name]``: + - if there's no value, :exc:`KeyError` is raised; + - if there's exactly one value, it's returned; + - if there's more than one value, :exc:`MultipleValuesError` is raised. + + - When setting a header with ``headers[name] = value``, the value is + appended to the list of values for that header. + + - When deleting a header with ``del headers[name]``, all values for that + header are removed (this is slow). + + Other methods for manipulating headers are consistent with this logic. + + As long as no header occurs multiple times, :class:`Headers` behaves like + :class:`dict`, except keys are lower-cased to provide case-insensitivity. + + Two methods support support manipulating multiple values explicitly: + + - :meth:`get_all` returns a list of all values for a header; + - :meth:`raw_items` returns an iterator of ``(name, values)`` pairs. + + """ + + __slots__ = ["_dict", "_list"] + + def __init__(self, *args: Any, **kwargs: str) -> None: + self._dict: Dict[str, List[str]] = {} + self._list: List[Tuple[str, str]] = [] + # MutableMapping.update calls __setitem__ for each (name, value) pair. + self.update(*args, **kwargs) + + def __str__(self) -> str: + return "".join(f"{key}: {value}\r\n" for key, value in self._list) + "\r\n" + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self._list!r})" + + def copy(self) -> "Headers": + copy = self.__class__() + copy._dict = self._dict.copy() + copy._list = self._list.copy() + return copy + + def serialize(self) -> bytes: + # Headers only contain ASCII characters. + return str(self).encode() + + # Collection methods + + def __contains__(self, key: object) -> bool: + return isinstance(key, str) and key.lower() in self._dict + + def __iter__(self) -> Iterator[str]: + return iter(self._dict) + + def __len__(self) -> int: + return len(self._dict) + + # MutableMapping methods + + def __getitem__(self, key: str) -> str: + value = self._dict[key.lower()] + if len(value) == 1: + return value[0] + else: + raise MultipleValuesError(key) + + def __setitem__(self, key: str, value: str) -> None: + self._dict.setdefault(key.lower(), []).append(value) + self._list.append((key, value)) + + def __delitem__(self, key: str) -> None: + key_lower = key.lower() + self._dict.__delitem__(key_lower) + # This is inefficent. Fortunately deleting HTTP headers is uncommon. + self._list = [(k, v) for k, v in self._list if k.lower() != key_lower] + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Headers): + return NotImplemented + return self._list == other._list + + def clear(self) -> None: + """ + Remove all headers. + + """ + self._dict = {} + self._list = [] + + # Methods for handling multiple values + + def get_all(self, key: str) -> List[str]: + """ + Return the (possibly empty) list of all values for a header. + + :param key: header name + + """ + return self._dict.get(key.lower(), []) + + def raw_items(self) -> Iterator[Tuple[str, str]]: + """ + Return an iterator of all values as ``(name, value)`` pairs. + + """ + return iter(self._list) + + +HeadersLike = Union[Headers, Mapping[str, str], Iterable[Tuple[str, str]]] diff --git a/src/websockets/exceptions.py b/src/websockets/exceptions.py index 9873a1717..e593f1adc 100644 --- a/src/websockets/exceptions.py +++ b/src/websockets/exceptions.py @@ -30,7 +30,7 @@ import http from typing import Optional -from .http import Headers, HeadersLike +from .datastructures import Headers, HeadersLike __all__ = [ diff --git a/src/websockets/handshake.py b/src/websockets/handshake.py index 646b6dba4..e30a67125 100644 --- a/src/websockets/handshake.py +++ b/src/websockets/handshake.py @@ -31,9 +31,9 @@ import random from typing import List +from .datastructures import Headers, MultipleValuesError from .exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade from .headers import ConnectionOption, UpgradeProtocol, parse_connection, parse_upgrade -from .http import Headers, MultipleValuesError __all__ = ["build_request", "check_request", "build_response", "check_response"] diff --git a/src/websockets/http.py b/src/websockets/http.py index f87bfb76a..ddb2afcfa 100644 --- a/src/websockets/http.py +++ b/src/websockets/http.py @@ -10,28 +10,15 @@ import asyncio import re import sys -from typing import ( - Any, - Dict, - Iterable, - Iterator, - List, - Mapping, - MutableMapping, - Tuple, - Union, -) +from typing import Tuple +# For backwards compatibility - should be deprecated +from .datastructures import Headers, MultipleValuesError # noqa +from .exceptions import SecurityError from .version import version as websockets_version -__all__ = [ - "read_request", - "read_response", - "Headers", - "MultipleValuesError", - "USER_AGENT", -] +__all__ = ["read_request", "read_response", "USER_AGENT"] MAX_HEADERS = 256 MAX_LINE = 4096 @@ -68,7 +55,7 @@ def d(value: bytes) -> str: _value_re = re.compile(rb"[\x09\x20-\x7e\x80-\xff]*") -async def read_request(stream: asyncio.StreamReader) -> Tuple[str, "Headers"]: +async def read_request(stream: asyncio.StreamReader) -> Tuple[str, Headers]: """ Read an HTTP/1.1 GET request and return ``(path, headers)``. @@ -114,7 +101,7 @@ async def read_request(stream: asyncio.StreamReader) -> Tuple[str, "Headers"]: return path, headers -async def read_response(stream: asyncio.StreamReader) -> Tuple[int, str, "Headers"]: +async def read_response(stream: asyncio.StreamReader) -> Tuple[int, str, Headers]: """ Read an HTTP/1.1 response and return ``(status_code, reason, headers)``. @@ -163,7 +150,7 @@ async def read_response(stream: asyncio.StreamReader) -> Tuple[int, str, "Header return status_code, reason, headers -async def read_headers(stream: asyncio.StreamReader) -> "Headers": +async def read_headers(stream: asyncio.StreamReader) -> Headers: """ Read HTTP headers from ``stream``. @@ -198,7 +185,7 @@ async def read_headers(stream: asyncio.StreamReader) -> "Headers": headers[name] = value else: - raise websockets.exceptions.SecurityError("too many HTTP headers") + raise SecurityError("too many HTTP headers") return headers @@ -214,148 +201,8 @@ async def read_line(stream: asyncio.StreamReader) -> bytes: line = await stream.readline() # Security: this guarantees header values are small (hard-coded = 4 KiB) if len(line) > MAX_LINE: - raise websockets.exceptions.SecurityError("line too long") + raise SecurityError("line too long") # Not mandatory but safe - https://tools.ietf.org/html/rfc7230#section-3.5 if not line.endswith(b"\r\n"): raise EOFError("line without CRLF") return line[:-2] - - -class MultipleValuesError(LookupError): - """ - Exception raised when :class:`Headers` has more than one value for a key. - - """ - - def __str__(self) -> str: - # Implement the same logic as KeyError_str in Objects/exceptions.c. - if len(self.args) == 1: - return repr(self.args[0]) - return super().__str__() - - -class Headers(MutableMapping[str, str]): - """ - Efficient data structure for manipulating HTTP headers. - - A :class:`list` of ``(name, values)`` is inefficient for lookups. - - A :class:`dict` doesn't suffice because header names are case-insensitive - and multiple occurrences of headers with the same name are possible. - - :class:`Headers` stores HTTP headers in a hybrid data structure to provide - efficient insertions and lookups while preserving the original data. - - In order to account for multiple values with minimal hassle, - :class:`Headers` follows this logic: - - - When getting a header with ``headers[name]``: - - if there's no value, :exc:`KeyError` is raised; - - if there's exactly one value, it's returned; - - if there's more than one value, :exc:`MultipleValuesError` is raised. - - - When setting a header with ``headers[name] = value``, the value is - appended to the list of values for that header. - - - When deleting a header with ``del headers[name]``, all values for that - header are removed (this is slow). - - Other methods for manipulating headers are consistent with this logic. - - As long as no header occurs multiple times, :class:`Headers` behaves like - :class:`dict`, except keys are lower-cased to provide case-insensitivity. - - Two methods support support manipulating multiple values explicitly: - - - :meth:`get_all` returns a list of all values for a header; - - :meth:`raw_items` returns an iterator of ``(name, values)`` pairs. - - """ - - __slots__ = ["_dict", "_list"] - - def __init__(self, *args: Any, **kwargs: str) -> None: - self._dict: Dict[str, List[str]] = {} - self._list: List[Tuple[str, str]] = [] - # MutableMapping.update calls __setitem__ for each (name, value) pair. - self.update(*args, **kwargs) - - def __str__(self) -> str: - return "".join(f"{key}: {value}\r\n" for key, value in self._list) + "\r\n" - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self._list!r})" - - def copy(self) -> "Headers": - copy = self.__class__() - copy._dict = self._dict.copy() - copy._list = self._list.copy() - return copy - - # Collection methods - - def __contains__(self, key: object) -> bool: - return isinstance(key, str) and key.lower() in self._dict - - def __iter__(self) -> Iterator[str]: - return iter(self._dict) - - def __len__(self) -> int: - return len(self._dict) - - # MutableMapping methods - - def __getitem__(self, key: str) -> str: - value = self._dict[key.lower()] - if len(value) == 1: - return value[0] - else: - raise MultipleValuesError(key) - - def __setitem__(self, key: str, value: str) -> None: - self._dict.setdefault(key.lower(), []).append(value) - self._list.append((key, value)) - - def __delitem__(self, key: str) -> None: - key_lower = key.lower() - self._dict.__delitem__(key_lower) - # This is inefficent. Fortunately deleting HTTP headers is uncommon. - self._list = [(k, v) for k, v in self._list if k.lower() != key_lower] - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Headers): - return NotImplemented - return self._list == other._list - - def clear(self) -> None: - """ - Remove all headers. - - """ - self._dict = {} - self._list = [] - - # Methods for handling multiple values - - def get_all(self, key: str) -> List[str]: - """ - Return the (possibly empty) list of all values for a header. - - :param key: header name - - """ - return self._dict.get(key.lower(), []) - - def raw_items(self) -> Iterator[Tuple[str, str]]: - """ - Return an iterator of all values as ``(name, value)`` pairs. - - """ - return iter(self._list) - - -HeadersLike = Union[Headers, Mapping[str, str], Iterable[Tuple[str, str]]] - - -# at the bottom to allow circular import, because AbortHandshake depends on HeadersLike -import websockets.exceptions # isort:skip # noqa diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 803970205..60235643e 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -30,6 +30,7 @@ cast, ) +from .datastructures import Headers from .exceptions import ( ConnectionClosed, ConnectionClosedError, @@ -41,7 +42,6 @@ from .extensions.base import Extension from .framing import * from .handshake import * -from .http import Headers from .typing import Data diff --git a/src/websockets/server.py b/src/websockets/server.py index 0f0b51a7c..da98cac05 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -28,6 +28,7 @@ cast, ) +from .datastructures import Headers, HeadersLike, MultipleValuesError from .exceptions import ( AbortHandshake, InvalidHandshake, @@ -41,7 +42,7 @@ from .extensions.permessage_deflate import ServerPerMessageDeflateFactory from .handshake import build_response, check_request from .headers import build_extension, parse_extension, parse_subprotocol -from .http import USER_AGENT, Headers, HeadersLike, MultipleValuesError, read_request +from .http import USER_AGENT, read_request from .protocol import WebSocketCommonProtocol from .typing import ExtensionHeader, Origin, Subprotocol diff --git a/tests/test_client_server.py b/tests/test_client_server.py index 35913666c..ba0984c80 100644 --- a/tests/test_client_server.py +++ b/tests/test_client_server.py @@ -14,6 +14,7 @@ import warnings from websockets.client import * +from websockets.datastructures import Headers from websockets.exceptions import ( ConnectionClosed, InvalidHandshake, @@ -27,7 +28,7 @@ ServerPerMessageDeflateFactory, ) from websockets.handshake import build_response -from websockets.http import USER_AGENT, Headers, read_response +from websockets.http import USER_AGENT, read_response from websockets.protocol import State from websockets.server import * from websockets.uri import parse_uri diff --git a/tests/test_datastructures.py b/tests/test_datastructures.py new file mode 100644 index 000000000..628cbcb02 --- /dev/null +++ b/tests/test_datastructures.py @@ -0,0 +1,131 @@ +import unittest + +from websockets.datastructures import * + + +class HeadersTests(unittest.TestCase): + def setUp(self): + self.headers = Headers([("Connection", "Upgrade"), ("Server", "websockets")]) + + def test_str(self): + self.assertEqual( + str(self.headers), "Connection: Upgrade\r\nServer: websockets\r\n\r\n" + ) + + def test_repr(self): + self.assertEqual( + repr(self.headers), + "Headers([('Connection', 'Upgrade'), ('Server', 'websockets')])", + ) + + def test_copy(self): + self.assertEqual(repr(self.headers.copy()), repr(self.headers)) + + def test_serialize(self): + self.assertEqual( + self.headers.serialize(), + b"Connection: Upgrade\r\nServer: websockets\r\n\r\n", + ) + + def test_multiple_values_error_str(self): + self.assertEqual(str(MultipleValuesError("Connection")), "'Connection'") + self.assertEqual(str(MultipleValuesError()), "") + + def test_contains(self): + self.assertIn("Server", self.headers) + + def test_contains_case_insensitive(self): + self.assertIn("server", self.headers) + + def test_contains_not_found(self): + self.assertNotIn("Date", self.headers) + + def test_contains_non_string_key(self): + self.assertNotIn(42, self.headers) + + def test_iter(self): + self.assertEqual(set(iter(self.headers)), {"connection", "server"}) + + def test_len(self): + self.assertEqual(len(self.headers), 2) + + def test_getitem(self): + self.assertEqual(self.headers["Server"], "websockets") + + def test_getitem_case_insensitive(self): + self.assertEqual(self.headers["server"], "websockets") + + def test_getitem_key_error(self): + with self.assertRaises(KeyError): + self.headers["Upgrade"] + + def test_getitem_multiple_values_error(self): + self.headers["Server"] = "2" + with self.assertRaises(MultipleValuesError): + self.headers["Server"] + + def test_setitem(self): + self.headers["Upgrade"] = "websocket" + self.assertEqual(self.headers["Upgrade"], "websocket") + + def test_setitem_case_insensitive(self): + self.headers["upgrade"] = "websocket" + self.assertEqual(self.headers["Upgrade"], "websocket") + + def test_setitem_multiple_values(self): + self.headers["Connection"] = "close" + with self.assertRaises(MultipleValuesError): + self.headers["Connection"] + + def test_delitem(self): + del self.headers["Connection"] + with self.assertRaises(KeyError): + self.headers["Connection"] + + def test_delitem_case_insensitive(self): + del self.headers["connection"] + with self.assertRaises(KeyError): + self.headers["Connection"] + + def test_delitem_multiple_values(self): + self.headers["Connection"] = "close" + del self.headers["Connection"] + with self.assertRaises(KeyError): + self.headers["Connection"] + + def test_eq(self): + other_headers = Headers([("Connection", "Upgrade"), ("Server", "websockets")]) + self.assertEqual(self.headers, other_headers) + + def test_eq_not_equal(self): + other_headers = Headers([("Connection", "close"), ("Server", "websockets")]) + self.assertNotEqual(self.headers, other_headers) + + def test_eq_other_type(self): + self.assertNotEqual( + self.headers, "Connection: Upgrade\r\nServer: websockets\r\n\r\n" + ) + + def test_clear(self): + self.headers.clear() + self.assertFalse(self.headers) + self.assertEqual(self.headers, Headers()) + + def test_get_all(self): + self.assertEqual(self.headers.get_all("Connection"), ["Upgrade"]) + + def test_get_all_case_insensitive(self): + self.assertEqual(self.headers.get_all("connection"), ["Upgrade"]) + + def test_get_all_no_values(self): + self.assertEqual(self.headers.get_all("Upgrade"), []) + + def test_get_all_multiple_values(self): + self.headers["Connection"] = "close" + self.assertEqual(self.headers.get_all("Connection"), ["Upgrade", "close"]) + + def test_raw_items(self): + self.assertEqual( + list(self.headers.raw_items()), + [("Connection", "Upgrade"), ("Server", "websockets")], + ) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 7ad5ad833..b800d4f91 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,7 +1,7 @@ import unittest +from websockets.datastructures import Headers from websockets.exceptions import * -from websockets.http import Headers class ExceptionsTests(unittest.TestCase): diff --git a/tests/test_handshake.py b/tests/test_handshake.py index 7d0477715..6850fec9a 100644 --- a/tests/test_handshake.py +++ b/tests/test_handshake.py @@ -1,6 +1,7 @@ import contextlib import unittest +from websockets.datastructures import Headers from websockets.exceptions import ( InvalidHandshake, InvalidHeader, @@ -9,7 +10,6 @@ ) from websockets.handshake import * from websockets.handshake import accept # private API -from websockets.http import Headers class HandshakeTests(unittest.TestCase): diff --git a/tests/test_http.py b/tests/test_http.py index 41b522c3d..b09247c3e 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,5 +1,4 @@ import asyncio -import unittest from websockets.exceptions import SecurityError from websockets.http import * @@ -134,116 +133,3 @@ async def test_line_ending(self): self.stream.feed_data(b"foo: bar\n\n") with self.assertRaises(EOFError): await read_headers(self.stream) - - -class HeadersTests(unittest.TestCase): - def setUp(self): - self.headers = Headers([("Connection", "Upgrade"), ("Server", USER_AGENT)]) - - def test_str(self): - self.assertEqual( - str(self.headers), f"Connection: Upgrade\r\nServer: {USER_AGENT}\r\n\r\n" - ) - - def test_repr(self): - self.assertEqual( - repr(self.headers), - f"Headers([('Connection', 'Upgrade'), " f"('Server', '{USER_AGENT}')])", - ) - - def test_multiple_values_error_str(self): - self.assertEqual(str(MultipleValuesError("Connection")), "'Connection'") - self.assertEqual(str(MultipleValuesError()), "") - - def test_contains(self): - self.assertIn("Server", self.headers) - - def test_contains_case_insensitive(self): - self.assertIn("server", self.headers) - - def test_contains_not_found(self): - self.assertNotIn("Date", self.headers) - - def test_contains_non_string_key(self): - self.assertNotIn(42, self.headers) - - def test_iter(self): - self.assertEqual(set(iter(self.headers)), {"connection", "server"}) - - def test_len(self): - self.assertEqual(len(self.headers), 2) - - def test_getitem(self): - self.assertEqual(self.headers["Server"], USER_AGENT) - - def test_getitem_case_insensitive(self): - self.assertEqual(self.headers["server"], USER_AGENT) - - def test_getitem_key_error(self): - with self.assertRaises(KeyError): - self.headers["Upgrade"] - - def test_getitem_multiple_values_error(self): - self.headers["Server"] = "2" - with self.assertRaises(MultipleValuesError): - self.headers["Server"] - - def test_setitem(self): - self.headers["Upgrade"] = "websocket" - self.assertEqual(self.headers["Upgrade"], "websocket") - - def test_setitem_case_insensitive(self): - self.headers["upgrade"] = "websocket" - self.assertEqual(self.headers["Upgrade"], "websocket") - - def test_setitem_multiple_values(self): - self.headers["Connection"] = "close" - with self.assertRaises(MultipleValuesError): - self.headers["Connection"] - - def test_delitem(self): - del self.headers["Connection"] - with self.assertRaises(KeyError): - self.headers["Connection"] - - def test_delitem_case_insensitive(self): - del self.headers["connection"] - with self.assertRaises(KeyError): - self.headers["Connection"] - - def test_delitem_multiple_values(self): - self.headers["Connection"] = "close" - del self.headers["Connection"] - with self.assertRaises(KeyError): - self.headers["Connection"] - - def test_eq(self): - other_headers = self.headers.copy() - self.assertEqual(self.headers, other_headers) - - def test_eq_not_equal(self): - self.assertNotEqual(self.headers, []) - - def test_clear(self): - self.headers.clear() - self.assertFalse(self.headers) - self.assertEqual(self.headers, Headers()) - - def test_get_all(self): - self.assertEqual(self.headers.get_all("Connection"), ["Upgrade"]) - - def test_get_all_case_insensitive(self): - self.assertEqual(self.headers.get_all("connection"), ["Upgrade"]) - - def test_get_all_no_values(self): - self.assertEqual(self.headers.get_all("Upgrade"), []) - - def test_get_all_multiple_values(self): - self.headers["Connection"] = "close" - self.assertEqual(self.headers.get_all("Connection"), ["Upgrade", "close"]) - - def test_raw_items(self): - self.assertEqual( - list(self.headers.raw_items()), - [("Connection", "Upgrade"), ("Server", USER_AGENT)], - ) From 1f19838c81c3bb30f94881143c43842ac09162ec Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 6 Oct 2019 12:19:20 +0200 Subject: [PATCH 020/104] Move the handshake and http modules out of the way. --- docs/api.rst | 9 -- docs/changelog.rst | 4 + src/websockets/client.py | 5 +- src/websockets/handshake.py | 191 ++++----------------------- src/websockets/handshake_legacy.py | 186 ++++++++++++++++++++++++++ src/websockets/http.py | 205 +++-------------------------- src/websockets/http_legacy.py | 193 +++++++++++++++++++++++++++ src/websockets/protocol.py | 2 +- src/websockets/server.py | 5 +- tests/test_client_server.py | 5 +- tests/test_handshake.py | 192 +-------------------------- tests/test_handshake_legacy.py | 190 ++++++++++++++++++++++++++ tests/test_http.py | 137 +------------------ tests/test_http_legacy.py | 135 +++++++++++++++++++ 14 files changed, 765 insertions(+), 694 deletions(-) create mode 100644 src/websockets/handshake_legacy.py create mode 100644 src/websockets/http_legacy.py create mode 100644 tests/test_handshake_legacy.py create mode 100644 tests/test_http_legacy.py diff --git a/docs/api.rst b/docs/api.rst index f7706ee2c..b4bddaf38 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -130,12 +130,6 @@ Exceptions Low-level --------- -Opening handshake -................. - -.. automodule:: websockets.handshake - :members: - Data transfer ............. @@ -153,6 +147,3 @@ Utilities .. automodule:: websockets.headers :members: - -.. automodule:: websockets.http - :members: diff --git a/docs/changelog.rst b/docs/changelog.rst index 5de7357ca..3cda4919f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,10 @@ Changelog :exc:`~datastructures.MultipleValuesError` from :mod:`websockets.datastructures` instead of :mod:`websockets.http`. + * :mod:`websockets.handshake` is deprecated. + + * :mod:`websockets.http` is deprecated. + Aliases provide backwards compatibility for all previously public APIs. 8.1 diff --git a/src/websockets/client.py b/src/websockets/client.py index 26a369c47..f95dae060 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -23,7 +23,7 @@ ) from .extensions.base import ClientExtensionFactory, Extension from .extensions.permessage_deflate import ClientPerMessageDeflateFactory -from .handshake import build_request, check_response +from .handshake_legacy import build_request, check_response from .headers import ( build_authorization_basic, build_extension, @@ -31,7 +31,8 @@ parse_extension, parse_subprotocol, ) -from .http import USER_AGENT, read_response +from .http import USER_AGENT +from .http_legacy import read_response from .protocol import WebSocketCommonProtocol from .typing import ExtensionHeader, Origin, Subprotocol from .uri import WebSocketURI, parse_uri diff --git a/src/websockets/handshake.py b/src/websockets/handshake.py index e30a67125..f27bd1b84 100644 --- a/src/websockets/handshake.py +++ b/src/websockets/handshake.py @@ -1,187 +1,48 @@ -""" -:mod:`websockets.handshake` provides helpers for the WebSocket handshake. +import warnings -See `section 4 of RFC 6455`_. - -.. _section 4 of RFC 6455: http://tools.ietf.org/html/rfc6455#section-4 - -Some checks cannot be performed because they depend too much on the -context; instead, they're documented below. - -To accept a connection, a server must: - -- Read the request, check that the method is GET, and check the headers with - :func:`check_request`, -- Send a 101 response to the client with the headers created by - :func:`build_response` if the request is valid; otherwise, send an - appropriate HTTP error code. - -To open a connection, a client must: - -- Send a GET request to the server with the headers created by - :func:`build_request`, -- Read the response, check that the status code is 101, and check the headers - with :func:`check_response`. - -""" - -import base64 -import binascii -import hashlib -import random -from typing import List - -from .datastructures import Headers, MultipleValuesError -from .exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade -from .headers import ConnectionOption, UpgradeProtocol, parse_connection, parse_upgrade +from .datastructures import Headers __all__ = ["build_request", "check_request", "build_response", "check_response"] -GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" - - -def build_request(headers: Headers) -> str: - """ - Build a handshake request to send to the server. - - Update request headers passed in argument. - - :param headers: request headers - :returns: ``key`` which must be passed to :func:`check_response` - - """ - raw_key = bytes(random.getrandbits(8) for _ in range(16)) - key = base64.b64encode(raw_key).decode() - headers["Upgrade"] = "websocket" - headers["Connection"] = "Upgrade" - headers["Sec-WebSocket-Key"] = key - headers["Sec-WebSocket-Version"] = "13" - return key +GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" -def check_request(headers: Headers) -> str: - """ - Check a handshake request received from the client. - This function doesn't verify that the request is an HTTP/1.1 or higher GET - request and doesn't perform ``Host`` and ``Origin`` checks. These controls - are usually performed earlier in the HTTP request handling code. They're - the responsibility of the caller. +# Backwards compatibility with previously documented public APIs - :param headers: request headers - :returns: ``key`` which must be passed to :func:`build_response` - :raises ~websockets.exceptions.InvalidHandshake: if the handshake request - is invalid; then the server must return 400 Bad Request error - """ - connection: List[ConnectionOption] = sum( - [parse_connection(value) for value in headers.get_all("Connection")], [] +def build_request(headers: Headers) -> str: # pragma: no cover + warnings.warn( + "websockets.handshake.build_request is deprecated", DeprecationWarning ) + from .handshake_legacy import build_request - if not any(value.lower() == "upgrade" for value in connection): - raise InvalidUpgrade("Connection", ", ".join(connection)) + return build_request(headers) - upgrade: List[UpgradeProtocol] = sum( - [parse_upgrade(value) for value in headers.get_all("Upgrade")], [] - ) - # For compatibility with non-strict implementations, ignore case when - # checking the Upgrade header. The RFC always uses "websocket", except - # in section 11.2. (IANA registration) where it uses "WebSocket". - if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): - raise InvalidUpgrade("Upgrade", ", ".join(upgrade)) - - try: - s_w_key = headers["Sec-WebSocket-Key"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Key") - except MultipleValuesError: - raise InvalidHeader( - "Sec-WebSocket-Key", "more than one Sec-WebSocket-Key header found" - ) - - try: - raw_key = base64.b64decode(s_w_key.encode(), validate=True) - except binascii.Error: - raise InvalidHeaderValue("Sec-WebSocket-Key", s_w_key) - if len(raw_key) != 16: - raise InvalidHeaderValue("Sec-WebSocket-Key", s_w_key) - - try: - s_w_version = headers["Sec-WebSocket-Version"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Version") - except MultipleValuesError: - raise InvalidHeader( - "Sec-WebSocket-Version", "more than one Sec-WebSocket-Version header found" - ) - - if s_w_version != "13": - raise InvalidHeaderValue("Sec-WebSocket-Version", s_w_version) - - return s_w_key - - -def build_response(headers: Headers, key: str) -> None: - """ - Build a handshake response to send to the client. - - Update response headers passed in argument. - - :param headers: response headers - :param key: comes from :func:`check_request` - - """ - headers["Upgrade"] = "websocket" - headers["Connection"] = "Upgrade" - headers["Sec-WebSocket-Accept"] = accept(key) - - -def check_response(headers: Headers, key: str) -> None: - """ - Check a handshake response received from the server. - - This function doesn't verify that the response is an HTTP/1.1 or higher - response with a 101 status code. These controls are the responsibility of - the caller. - - :param headers: response headers - :param key: comes from :func:`build_request` - :raises ~websockets.exceptions.InvalidHandshake: if the handshake response - is invalid - - """ - connection: List[ConnectionOption] = sum( - [parse_connection(value) for value in headers.get_all("Connection")], [] +def check_request(headers: Headers) -> str: # pragma: no cover + warnings.warn( + "websockets.handshake.check_request is deprecated", DeprecationWarning ) + from .handshake_legacy import check_request - if not any(value.lower() == "upgrade" for value in connection): - raise InvalidUpgrade("Connection", " ".join(connection)) + return check_request(headers) - upgrade: List[UpgradeProtocol] = sum( - [parse_upgrade(value) for value in headers.get_all("Upgrade")], [] - ) - # For compatibility with non-strict implementations, ignore case when - # checking the Upgrade header. The RFC always uses "websocket", except - # in section 11.2. (IANA registration) where it uses "WebSocket". - if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): - raise InvalidUpgrade("Upgrade", ", ".join(upgrade)) +def build_response(headers: Headers, key: str) -> None: # pragma: no cover + warnings.warn( + "websockets.handshake.build_response is deprecated", DeprecationWarning + ) + from .handshake_legacy import build_response - try: - s_w_accept = headers["Sec-WebSocket-Accept"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Accept") - except MultipleValuesError: - raise InvalidHeader( - "Sec-WebSocket-Accept", "more than one Sec-WebSocket-Accept header found" - ) + return build_response(headers, key) - if s_w_accept != accept(key): - raise InvalidHeaderValue("Sec-WebSocket-Accept", s_w_accept) +def check_response(headers: Headers, key: str) -> None: # pragma: no cover + warnings.warn( + "websockets.handshake.check_response is deprecated", DeprecationWarning + ) + from .handshake_legacy import check_response -def accept(key: str) -> str: - sha1 = hashlib.sha1((key + GUID).encode()).digest() - return base64.b64encode(sha1).decode() + return check_response(headers, key) diff --git a/src/websockets/handshake_legacy.py b/src/websockets/handshake_legacy.py new file mode 100644 index 000000000..3fca45545 --- /dev/null +++ b/src/websockets/handshake_legacy.py @@ -0,0 +1,186 @@ +""" +:mod:`websockets.handshake` provides helpers for the WebSocket handshake. + +See `section 4 of RFC 6455`_. + +.. _section 4 of RFC 6455: http://tools.ietf.org/html/rfc6455#section-4 + +Some checks cannot be performed because they depend too much on the +context; instead, they're documented below. + +To accept a connection, a server must: + +- Read the request, check that the method is GET, and check the headers with + :func:`check_request`, +- Send a 101 response to the client with the headers created by + :func:`build_response` if the request is valid; otherwise, send an + appropriate HTTP error code. + +To open a connection, a client must: + +- Send a GET request to the server with the headers created by + :func:`build_request`, +- Read the response, check that the status code is 101, and check the headers + with :func:`check_response`. + +""" + +import base64 +import binascii +import hashlib +import random +from typing import List + +from .datastructures import Headers, MultipleValuesError +from .exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade +from .handshake import GUID +from .headers import ConnectionOption, UpgradeProtocol, parse_connection, parse_upgrade + + +__all__ = ["build_request", "check_request", "build_response", "check_response"] + + +def build_request(headers: Headers) -> str: + """ + Build a handshake request to send to the server. + + Update request headers passed in argument. + + :param headers: request headers + :returns: ``key`` which must be passed to :func:`check_response` + + """ + raw_key = bytes(random.getrandbits(8) for _ in range(16)) + key = base64.b64encode(raw_key).decode() + headers["Upgrade"] = "websocket" + headers["Connection"] = "Upgrade" + headers["Sec-WebSocket-Key"] = key + headers["Sec-WebSocket-Version"] = "13" + return key + + +def check_request(headers: Headers) -> str: + """ + Check a handshake request received from the client. + + This function doesn't verify that the request is an HTTP/1.1 or higher GET + request and doesn't perform ``Host`` and ``Origin`` checks. These controls + are usually performed earlier in the HTTP request handling code. They're + the responsibility of the caller. + + :param headers: request headers + :returns: ``key`` which must be passed to :func:`build_response` + :raises ~websockets.exceptions.InvalidHandshake: if the handshake request + is invalid; then the server must return 400 Bad Request error + + """ + connection: List[ConnectionOption] = sum( + [parse_connection(value) for value in headers.get_all("Connection")], [] + ) + + if not any(value.lower() == "upgrade" for value in connection): + raise InvalidUpgrade("Connection", ", ".join(connection)) + + upgrade: List[UpgradeProtocol] = sum( + [parse_upgrade(value) for value in headers.get_all("Upgrade")], [] + ) + + # For compatibility with non-strict implementations, ignore case when + # checking the Upgrade header. The RFC always uses "websocket", except + # in section 11.2. (IANA registration) where it uses "WebSocket". + if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): + raise InvalidUpgrade("Upgrade", ", ".join(upgrade)) + + try: + s_w_key = headers["Sec-WebSocket-Key"] + except KeyError: + raise InvalidHeader("Sec-WebSocket-Key") + except MultipleValuesError: + raise InvalidHeader( + "Sec-WebSocket-Key", "more than one Sec-WebSocket-Key header found" + ) + + try: + raw_key = base64.b64decode(s_w_key.encode(), validate=True) + except binascii.Error: + raise InvalidHeaderValue("Sec-WebSocket-Key", s_w_key) + if len(raw_key) != 16: + raise InvalidHeaderValue("Sec-WebSocket-Key", s_w_key) + + try: + s_w_version = headers["Sec-WebSocket-Version"] + except KeyError: + raise InvalidHeader("Sec-WebSocket-Version") + except MultipleValuesError: + raise InvalidHeader( + "Sec-WebSocket-Version", "more than one Sec-WebSocket-Version header found" + ) + + if s_w_version != "13": + raise InvalidHeaderValue("Sec-WebSocket-Version", s_w_version) + + return s_w_key + + +def build_response(headers: Headers, key: str) -> None: + """ + Build a handshake response to send to the client. + + Update response headers passed in argument. + + :param headers: response headers + :param key: comes from :func:`check_request` + + """ + headers["Upgrade"] = "websocket" + headers["Connection"] = "Upgrade" + headers["Sec-WebSocket-Accept"] = accept(key) + + +def check_response(headers: Headers, key: str) -> None: + """ + Check a handshake response received from the server. + + This function doesn't verify that the response is an HTTP/1.1 or higher + response with a 101 status code. These controls are the responsibility of + the caller. + + :param headers: response headers + :param key: comes from :func:`build_request` + :raises ~websockets.exceptions.InvalidHandshake: if the handshake response + is invalid + + """ + connection: List[ConnectionOption] = sum( + [parse_connection(value) for value in headers.get_all("Connection")], [] + ) + + if not any(value.lower() == "upgrade" for value in connection): + raise InvalidUpgrade("Connection", " ".join(connection)) + + upgrade: List[UpgradeProtocol] = sum( + [parse_upgrade(value) for value in headers.get_all("Upgrade")], [] + ) + + # For compatibility with non-strict implementations, ignore case when + # checking the Upgrade header. The RFC always uses "websocket", except + # in section 11.2. (IANA registration) where it uses "WebSocket". + if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): + raise InvalidUpgrade("Upgrade", ", ".join(upgrade)) + + try: + s_w_accept = headers["Sec-WebSocket-Accept"] + except KeyError: + raise InvalidHeader("Sec-WebSocket-Accept") + except MultipleValuesError: + raise InvalidHeader( + "Sec-WebSocket-Accept", "more than one Sec-WebSocket-Accept header found" + ) + + if s_w_accept != accept(key): + raise InvalidHeaderValue("Sec-WebSocket-Accept", s_w_accept) + + +def accept(key: str) -> str: + sha1 = hashlib.sha1((key + GUID).encode()).digest() + return base64.b64encode(sha1).decode() diff --git a/src/websockets/http.py b/src/websockets/http.py index ddb2afcfa..850b9beaa 100644 --- a/src/websockets/http.py +++ b/src/websockets/http.py @@ -1,208 +1,37 @@ -""" -:mod:`websockets.http` module provides basic HTTP/1.1 support. It is merely -:adequate for WebSocket handshake messages. - -These APIs cannot be imported from :mod:`websockets`. They must be imported -from :mod:`websockets.http`. - -""" - import asyncio -import re import sys +import warnings from typing import Tuple -# For backwards compatibility - should be deprecated +# For backwards compatibility: +# Headers and MultipleValuesError used to be defined in this module from .datastructures import Headers, MultipleValuesError # noqa -from .exceptions import SecurityError from .version import version as websockets_version -__all__ = ["read_request", "read_response", "USER_AGENT"] +__all__ = ["USER_AGENT"] -MAX_HEADERS = 256 -MAX_LINE = 4096 PYTHON_VERSION = "{}.{}".format(*sys.version_info) USER_AGENT = f"Python/{PYTHON_VERSION} websockets/{websockets_version}" -def d(value: bytes) -> str: - """ - Decode a bytestring for interpolating into an error message. - - """ - return value.decode(errors="backslashreplace") - - -# See https://tools.ietf.org/html/rfc7230#appendix-B. - -# Regex for validating header names. - -_token_re = re.compile(rb"[-!#$%&\'*+.^_`|~0-9a-zA-Z]+") - -# Regex for validating header values. - -# We don't attempt to support obsolete line folding. - -# Include HTAB (\x09), SP (\x20), VCHAR (\x21-\x7e), obs-text (\x80-\xff). - -# The ABNF is complicated because it attempts to express that optional -# whitespace is ignored. We strip whitespace and don't revalidate that. - -# See also https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 - -_value_re = re.compile(rb"[\x09\x20-\x7e\x80-\xff]*") - - -async def read_request(stream: asyncio.StreamReader) -> Tuple[str, Headers]: - """ - Read an HTTP/1.1 GET request and return ``(path, headers)``. - - ``path`` isn't URL-decoded or validated in any way. - - ``path`` and ``headers`` are expected to contain only ASCII characters. - Other characters are represented with surrogate escapes. - - :func:`read_request` doesn't attempt to read the request body because - WebSocket handshake requests don't have one. If the request contains a - body, it may be read from ``stream`` after this coroutine returns. - - :param stream: input to read the request from - :raises EOFError: if the connection is closed without a full HTTP request - :raises SecurityError: if the request exceeds a security limit - :raises ValueError: if the request isn't well formatted - - """ - # https://tools.ietf.org/html/rfc7230#section-3.1.1 - - # Parsing is simple because fixed values are expected for method and - # version and because path isn't checked. Since WebSocket software tends - # to implement HTTP/1.1 strictly, there's little need for lenient parsing. - - try: - request_line = await read_line(stream) - except EOFError as exc: - raise EOFError("connection closed while reading HTTP request line") from exc - - try: - method, raw_path, version = request_line.split(b" ", 2) - except ValueError: # not enough values to unpack (expected 3, got 1-2) - raise ValueError(f"invalid HTTP request line: {d(request_line)}") from None - - if method != b"GET": - raise ValueError(f"unsupported HTTP method: {d(method)}") - if version != b"HTTP/1.1": - raise ValueError(f"unsupported HTTP version: {d(version)}") - path = raw_path.decode("ascii", "surrogateescape") - - headers = await read_headers(stream) - - return path, headers - - -async def read_response(stream: asyncio.StreamReader) -> Tuple[int, str, Headers]: - """ - Read an HTTP/1.1 response and return ``(status_code, reason, headers)``. - - ``reason`` and ``headers`` are expected to contain only ASCII characters. - Other characters are represented with surrogate escapes. - - :func:`read_request` doesn't attempt to read the response body because - WebSocket handshake responses don't have one. If the response contains a - body, it may be read from ``stream`` after this coroutine returns. - - :param stream: input to read the response from - :raises EOFError: if the connection is closed without a full HTTP response - :raises SecurityError: if the response exceeds a security limit - :raises ValueError: if the response isn't well formatted - - """ - # https://tools.ietf.org/html/rfc7230#section-3.1.2 - - # As in read_request, parsing is simple because a fixed value is expected - # for version, status_code is a 3-digit number, and reason can be ignored. - - try: - status_line = await read_line(stream) - except EOFError as exc: - raise EOFError("connection closed while reading HTTP status line") from exc - - try: - version, raw_status_code, raw_reason = status_line.split(b" ", 2) - except ValueError: # not enough values to unpack (expected 3, got 1-2) - raise ValueError(f"invalid HTTP status line: {d(status_line)}") from None - - if version != b"HTTP/1.1": - raise ValueError(f"unsupported HTTP version: {d(version)}") - try: - status_code = int(raw_status_code) - except ValueError: # invalid literal for int() with base 10 - raise ValueError(f"invalid HTTP status code: {d(raw_status_code)}") from None - if not 100 <= status_code < 1000: - raise ValueError(f"unsupported HTTP status code: {d(raw_status_code)}") - if not _value_re.fullmatch(raw_reason): - raise ValueError(f"invalid HTTP reason phrase: {d(raw_reason)}") - reason = raw_reason.decode() - - headers = await read_headers(stream) - - return status_code, reason, headers - - -async def read_headers(stream: asyncio.StreamReader) -> Headers: - """ - Read HTTP headers from ``stream``. - - Non-ASCII characters are represented with surrogate escapes. - - """ - # https://tools.ietf.org/html/rfc7230#section-3.2 - - # We don't attempt to support obsolete line folding. - - headers = Headers() - for _ in range(MAX_HEADERS + 1): - try: - line = await read_line(stream) - except EOFError as exc: - raise EOFError("connection closed while reading HTTP headers") from exc - if line == b"": - break - - try: - raw_name, raw_value = line.split(b":", 1) - except ValueError: # not enough values to unpack (expected 2, got 1) - raise ValueError(f"invalid HTTP header line: {d(line)}") from None - if not _token_re.fullmatch(raw_name): - raise ValueError(f"invalid HTTP header name: {d(raw_name)}") - raw_value = raw_value.strip(b" \t") - if not _value_re.fullmatch(raw_value): - raise ValueError(f"invalid HTTP header value: {d(raw_value)}") - - name = raw_name.decode("ascii") # guaranteed to be ASCII at this point - value = raw_value.decode("ascii", "surrogateescape") - headers[name] = value +# Backwards compatibility with previously documented public APIs - else: - raise SecurityError("too many HTTP headers") - return headers +async def read_request( + stream: asyncio.StreamReader, +) -> Tuple[str, Headers]: # pragma: no cover + warnings.warn("websockets.http.read_request is deprecated", DeprecationWarning) + from .http_legacy import read_request + return await read_request(stream) -async def read_line(stream: asyncio.StreamReader) -> bytes: - """ - Read a single line from ``stream``. - CRLF is stripped from the return value. +async def read_response( + stream: asyncio.StreamReader, +) -> Tuple[int, str, Headers]: # pragma: no cover + warnings.warn("websockets.http.read_response is deprecated", DeprecationWarning) + from .http_legacy import read_response - """ - # Security: this is bounded by the StreamReader's limit (default = 32 KiB). - line = await stream.readline() - # Security: this guarantees header values are small (hard-coded = 4 KiB) - if len(line) > MAX_LINE: - raise SecurityError("line too long") - # Not mandatory but safe - https://tools.ietf.org/html/rfc7230#section-3.5 - if not line.endswith(b"\r\n"): - raise EOFError("line without CRLF") - return line[:-2] + return await read_response(stream) diff --git a/src/websockets/http_legacy.py b/src/websockets/http_legacy.py new file mode 100644 index 000000000..3630d3593 --- /dev/null +++ b/src/websockets/http_legacy.py @@ -0,0 +1,193 @@ +import asyncio +import re +from typing import Tuple + +from .datastructures import Headers +from .exceptions import SecurityError + + +__all__ = ["read_request", "read_response"] + +MAX_HEADERS = 256 +MAX_LINE = 4096 + + +def d(value: bytes) -> str: + """ + Decode a bytestring for interpolating into an error message. + + """ + return value.decode(errors="backslashreplace") + + +# See https://tools.ietf.org/html/rfc7230#appendix-B. + +# Regex for validating header names. + +_token_re = re.compile(rb"[-!#$%&\'*+.^_`|~0-9a-zA-Z]+") + +# Regex for validating header values. + +# We don't attempt to support obsolete line folding. + +# Include HTAB (\x09), SP (\x20), VCHAR (\x21-\x7e), obs-text (\x80-\xff). + +# The ABNF is complicated because it attempts to express that optional +# whitespace is ignored. We strip whitespace and don't revalidate that. + +# See also https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 + +_value_re = re.compile(rb"[\x09\x20-\x7e\x80-\xff]*") + + +async def read_request(stream: asyncio.StreamReader) -> Tuple[str, Headers]: + """ + Read an HTTP/1.1 GET request and return ``(path, headers)``. + + ``path`` isn't URL-decoded or validated in any way. + + ``path`` and ``headers`` are expected to contain only ASCII characters. + Other characters are represented with surrogate escapes. + + :func:`read_request` doesn't attempt to read the request body because + WebSocket handshake requests don't have one. If the request contains a + body, it may be read from ``stream`` after this coroutine returns. + + :param stream: input to read the request from + :raises EOFError: if the connection is closed without a full HTTP request + :raises SecurityError: if the request exceeds a security limit + :raises ValueError: if the request isn't well formatted + + """ + # https://tools.ietf.org/html/rfc7230#section-3.1.1 + + # Parsing is simple because fixed values are expected for method and + # version and because path isn't checked. Since WebSocket software tends + # to implement HTTP/1.1 strictly, there's little need for lenient parsing. + + try: + request_line = await read_line(stream) + except EOFError as exc: + raise EOFError("connection closed while reading HTTP request line") from exc + + try: + method, raw_path, version = request_line.split(b" ", 2) + except ValueError: # not enough values to unpack (expected 3, got 1-2) + raise ValueError(f"invalid HTTP request line: {d(request_line)}") from None + + if method != b"GET": + raise ValueError(f"unsupported HTTP method: {d(method)}") + if version != b"HTTP/1.1": + raise ValueError(f"unsupported HTTP version: {d(version)}") + path = raw_path.decode("ascii", "surrogateescape") + + headers = await read_headers(stream) + + return path, headers + + +async def read_response(stream: asyncio.StreamReader) -> Tuple[int, str, Headers]: + """ + Read an HTTP/1.1 response and return ``(status_code, reason, headers)``. + + ``reason`` and ``headers`` are expected to contain only ASCII characters. + Other characters are represented with surrogate escapes. + + :func:`read_request` doesn't attempt to read the response body because + WebSocket handshake responses don't have one. If the response contains a + body, it may be read from ``stream`` after this coroutine returns. + + :param stream: input to read the response from + :raises EOFError: if the connection is closed without a full HTTP response + :raises SecurityError: if the response exceeds a security limit + :raises ValueError: if the response isn't well formatted + + """ + # https://tools.ietf.org/html/rfc7230#section-3.1.2 + + # As in read_request, parsing is simple because a fixed value is expected + # for version, status_code is a 3-digit number, and reason can be ignored. + + try: + status_line = await read_line(stream) + except EOFError as exc: + raise EOFError("connection closed while reading HTTP status line") from exc + + try: + version, raw_status_code, raw_reason = status_line.split(b" ", 2) + except ValueError: # not enough values to unpack (expected 3, got 1-2) + raise ValueError(f"invalid HTTP status line: {d(status_line)}") from None + + if version != b"HTTP/1.1": + raise ValueError(f"unsupported HTTP version: {d(version)}") + try: + status_code = int(raw_status_code) + except ValueError: # invalid literal for int() with base 10 + raise ValueError(f"invalid HTTP status code: {d(raw_status_code)}") from None + if not 100 <= status_code < 1000: + raise ValueError(f"unsupported HTTP status code: {d(raw_status_code)}") + if not _value_re.fullmatch(raw_reason): + raise ValueError(f"invalid HTTP reason phrase: {d(raw_reason)}") + reason = raw_reason.decode() + + headers = await read_headers(stream) + + return status_code, reason, headers + + +async def read_headers(stream: asyncio.StreamReader) -> Headers: + """ + Read HTTP headers from ``stream``. + + Non-ASCII characters are represented with surrogate escapes. + + """ + # https://tools.ietf.org/html/rfc7230#section-3.2 + + # We don't attempt to support obsolete line folding. + + headers = Headers() + for _ in range(MAX_HEADERS + 1): + try: + line = await read_line(stream) + except EOFError as exc: + raise EOFError("connection closed while reading HTTP headers") from exc + if line == b"": + break + + try: + raw_name, raw_value = line.split(b":", 1) + except ValueError: # not enough values to unpack (expected 2, got 1) + raise ValueError(f"invalid HTTP header line: {d(line)}") from None + if not _token_re.fullmatch(raw_name): + raise ValueError(f"invalid HTTP header name: {d(raw_name)}") + raw_value = raw_value.strip(b" \t") + if not _value_re.fullmatch(raw_value): + raise ValueError(f"invalid HTTP header value: {d(raw_value)}") + + name = raw_name.decode("ascii") # guaranteed to be ASCII at this point + value = raw_value.decode("ascii", "surrogateescape") + headers[name] = value + + else: + raise SecurityError("too many HTTP headers") + + return headers + + +async def read_line(stream: asyncio.StreamReader) -> bytes: + """ + Read a single line from ``stream``. + + CRLF is stripped from the return value. + + """ + # Security: this is bounded by the StreamReader's limit (default = 32 KiB). + line = await stream.readline() + # Security: this guarantees header values are small (hard-coded = 4 KiB) + if len(line) > MAX_LINE: + raise SecurityError("line too long") + # Not mandatory but safe - https://tools.ietf.org/html/rfc7230#section-3.5 + if not line.endswith(b"\r\n"): + raise EOFError("line without CRLF") + return line[:-2] diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 60235643e..cc4416ba8 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -41,7 +41,7 @@ ) from .extensions.base import Extension from .framing import * -from .handshake import * +from .handshake_legacy import * from .typing import Data diff --git a/src/websockets/server.py b/src/websockets/server.py index da98cac05..522c76114 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -40,9 +40,10 @@ ) from .extensions.base import Extension, ServerExtensionFactory from .extensions.permessage_deflate import ServerPerMessageDeflateFactory -from .handshake import build_response, check_request +from .handshake_legacy import build_response, check_request from .headers import build_extension, parse_extension, parse_subprotocol -from .http import USER_AGENT, read_request +from .http import USER_AGENT +from .http_legacy import read_request from .protocol import WebSocketCommonProtocol from .typing import ExtensionHeader, Origin, Subprotocol diff --git a/tests/test_client_server.py b/tests/test_client_server.py index ba0984c80..db26d6583 100644 --- a/tests/test_client_server.py +++ b/tests/test_client_server.py @@ -27,8 +27,9 @@ PerMessageDeflate, ServerPerMessageDeflateFactory, ) -from websockets.handshake import build_response -from websockets.http import USER_AGENT, read_response +from websockets.handshake_legacy import build_response +from websockets.http import USER_AGENT +from websockets.http_legacy import read_response from websockets.protocol import State from websockets.server import * from websockets.uri import parse_uri diff --git a/tests/test_handshake.py b/tests/test_handshake.py index 6850fec9a..8c35c9714 100644 --- a/tests/test_handshake.py +++ b/tests/test_handshake.py @@ -1,190 +1,2 @@ -import contextlib -import unittest - -from websockets.datastructures import Headers -from websockets.exceptions import ( - InvalidHandshake, - InvalidHeader, - InvalidHeaderValue, - InvalidUpgrade, -) -from websockets.handshake import * -from websockets.handshake import accept # private API - - -class HandshakeTests(unittest.TestCase): - def test_accept(self): - # Test vector from RFC 6455 - key = "dGhlIHNhbXBsZSBub25jZQ==" - acc = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" - self.assertEqual(accept(key), acc) - - def test_round_trip(self): - request_headers = Headers() - request_key = build_request(request_headers) - response_key = check_request(request_headers) - self.assertEqual(request_key, response_key) - response_headers = Headers() - build_response(response_headers, response_key) - check_response(response_headers, request_key) - - @contextlib.contextmanager - def assertValidRequestHeaders(self): - """ - Provide request headers for modification. - - Assert that the transformation kept them valid. - - """ - headers = Headers() - build_request(headers) - yield headers - check_request(headers) - - @contextlib.contextmanager - def assertInvalidRequestHeaders(self, exc_type): - """ - Provide request headers for modification. - - Assert that the transformation made them invalid. - - """ - headers = Headers() - build_request(headers) - yield headers - assert issubclass(exc_type, InvalidHandshake) - with self.assertRaises(exc_type): - check_request(headers) - - def test_request_invalid_connection(self): - with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: - del headers["Connection"] - headers["Connection"] = "Downgrade" - - def test_request_missing_connection(self): - with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: - del headers["Connection"] - - def test_request_additional_connection(self): - with self.assertValidRequestHeaders() as headers: - headers["Connection"] = "close" - - def test_request_invalid_upgrade(self): - with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: - del headers["Upgrade"] - headers["Upgrade"] = "socketweb" - - def test_request_missing_upgrade(self): - with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: - del headers["Upgrade"] - - def test_request_additional_upgrade(self): - with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: - headers["Upgrade"] = "socketweb" - - def test_request_invalid_key_not_base64(self): - with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: - del headers["Sec-WebSocket-Key"] - headers["Sec-WebSocket-Key"] = "!@#$%^&*()" - - def test_request_invalid_key_not_well_padded(self): - with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: - del headers["Sec-WebSocket-Key"] - headers["Sec-WebSocket-Key"] = "CSIRmL8dWYxeAdr/XpEHRw" - - def test_request_invalid_key_not_16_bytes_long(self): - with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: - del headers["Sec-WebSocket-Key"] - headers["Sec-WebSocket-Key"] = "ZLpprpvK4PE=" - - def test_request_missing_key(self): - with self.assertInvalidRequestHeaders(InvalidHeader) as headers: - del headers["Sec-WebSocket-Key"] - - def test_request_additional_key(self): - with self.assertInvalidRequestHeaders(InvalidHeader) as headers: - # This duplicates the Sec-WebSocket-Key header. - headers["Sec-WebSocket-Key"] = headers["Sec-WebSocket-Key"] - - def test_request_invalid_version(self): - with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: - del headers["Sec-WebSocket-Version"] - headers["Sec-WebSocket-Version"] = "42" - - def test_request_missing_version(self): - with self.assertInvalidRequestHeaders(InvalidHeader) as headers: - del headers["Sec-WebSocket-Version"] - - def test_request_additional_version(self): - with self.assertInvalidRequestHeaders(InvalidHeader) as headers: - # This duplicates the Sec-WebSocket-Version header. - headers["Sec-WebSocket-Version"] = headers["Sec-WebSocket-Version"] - - @contextlib.contextmanager - def assertValidResponseHeaders(self, key="CSIRmL8dWYxeAdr/XpEHRw=="): - """ - Provide response headers for modification. - - Assert that the transformation kept them valid. - - """ - headers = Headers() - build_response(headers, key) - yield headers - check_response(headers, key) - - @contextlib.contextmanager - def assertInvalidResponseHeaders(self, exc_type, key="CSIRmL8dWYxeAdr/XpEHRw=="): - """ - Provide response headers for modification. - - Assert that the transformation made them invalid. - - """ - headers = Headers() - build_response(headers, key) - yield headers - assert issubclass(exc_type, InvalidHandshake) - with self.assertRaises(exc_type): - check_response(headers, key) - - def test_response_invalid_connection(self): - with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: - del headers["Connection"] - headers["Connection"] = "Downgrade" - - def test_response_missing_connection(self): - with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: - del headers["Connection"] - - def test_response_additional_connection(self): - with self.assertValidResponseHeaders() as headers: - headers["Connection"] = "close" - - def test_response_invalid_upgrade(self): - with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: - del headers["Upgrade"] - headers["Upgrade"] = "socketweb" - - def test_response_missing_upgrade(self): - with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: - del headers["Upgrade"] - - def test_response_additional_upgrade(self): - with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: - headers["Upgrade"] = "socketweb" - - def test_response_invalid_accept(self): - with self.assertInvalidResponseHeaders(InvalidHeaderValue) as headers: - del headers["Sec-WebSocket-Accept"] - other_key = "1Eq4UDEFQYg3YspNgqxv5g==" - headers["Sec-WebSocket-Accept"] = accept(other_key) - - def test_response_missing_accept(self): - with self.assertInvalidResponseHeaders(InvalidHeader) as headers: - del headers["Sec-WebSocket-Accept"] - - def test_response_additional_accept(self): - with self.assertInvalidResponseHeaders(InvalidHeader) as headers: - # This duplicates the Sec-WebSocket-Accept header. - headers["Sec-WebSocket-Accept"] = headers["Sec-WebSocket-Accept"] +# Check that the legacy handshake module imports without an exception. +from websockets.handshake import * # noqa diff --git a/tests/test_handshake_legacy.py b/tests/test_handshake_legacy.py new file mode 100644 index 000000000..361410d3f --- /dev/null +++ b/tests/test_handshake_legacy.py @@ -0,0 +1,190 @@ +import contextlib +import unittest + +from websockets.datastructures import Headers +from websockets.exceptions import ( + InvalidHandshake, + InvalidHeader, + InvalidHeaderValue, + InvalidUpgrade, +) +from websockets.handshake_legacy import * +from websockets.handshake_legacy import accept # private API + + +class HandshakeTests(unittest.TestCase): + def test_accept(self): + # Test vector from RFC 6455 + key = "dGhlIHNhbXBsZSBub25jZQ==" + acc = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" + self.assertEqual(accept(key), acc) + + def test_round_trip(self): + request_headers = Headers() + request_key = build_request(request_headers) + response_key = check_request(request_headers) + self.assertEqual(request_key, response_key) + response_headers = Headers() + build_response(response_headers, response_key) + check_response(response_headers, request_key) + + @contextlib.contextmanager + def assertValidRequestHeaders(self): + """ + Provide request headers for modification. + + Assert that the transformation kept them valid. + + """ + headers = Headers() + build_request(headers) + yield headers + check_request(headers) + + @contextlib.contextmanager + def assertInvalidRequestHeaders(self, exc_type): + """ + Provide request headers for modification. + + Assert that the transformation made them invalid. + + """ + headers = Headers() + build_request(headers) + yield headers + assert issubclass(exc_type, InvalidHandshake) + with self.assertRaises(exc_type): + check_request(headers) + + def test_request_invalid_connection(self): + with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: + del headers["Connection"] + headers["Connection"] = "Downgrade" + + def test_request_missing_connection(self): + with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: + del headers["Connection"] + + def test_request_additional_connection(self): + with self.assertValidRequestHeaders() as headers: + headers["Connection"] = "close" + + def test_request_invalid_upgrade(self): + with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: + del headers["Upgrade"] + headers["Upgrade"] = "socketweb" + + def test_request_missing_upgrade(self): + with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: + del headers["Upgrade"] + + def test_request_additional_upgrade(self): + with self.assertInvalidRequestHeaders(InvalidUpgrade) as headers: + headers["Upgrade"] = "socketweb" + + def test_request_invalid_key_not_base64(self): + with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: + del headers["Sec-WebSocket-Key"] + headers["Sec-WebSocket-Key"] = "!@#$%^&*()" + + def test_request_invalid_key_not_well_padded(self): + with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: + del headers["Sec-WebSocket-Key"] + headers["Sec-WebSocket-Key"] = "CSIRmL8dWYxeAdr/XpEHRw" + + def test_request_invalid_key_not_16_bytes_long(self): + with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: + del headers["Sec-WebSocket-Key"] + headers["Sec-WebSocket-Key"] = "ZLpprpvK4PE=" + + def test_request_missing_key(self): + with self.assertInvalidRequestHeaders(InvalidHeader) as headers: + del headers["Sec-WebSocket-Key"] + + def test_request_additional_key(self): + with self.assertInvalidRequestHeaders(InvalidHeader) as headers: + # This duplicates the Sec-WebSocket-Key header. + headers["Sec-WebSocket-Key"] = headers["Sec-WebSocket-Key"] + + def test_request_invalid_version(self): + with self.assertInvalidRequestHeaders(InvalidHeaderValue) as headers: + del headers["Sec-WebSocket-Version"] + headers["Sec-WebSocket-Version"] = "42" + + def test_request_missing_version(self): + with self.assertInvalidRequestHeaders(InvalidHeader) as headers: + del headers["Sec-WebSocket-Version"] + + def test_request_additional_version(self): + with self.assertInvalidRequestHeaders(InvalidHeader) as headers: + # This duplicates the Sec-WebSocket-Version header. + headers["Sec-WebSocket-Version"] = headers["Sec-WebSocket-Version"] + + @contextlib.contextmanager + def assertValidResponseHeaders(self, key="CSIRmL8dWYxeAdr/XpEHRw=="): + """ + Provide response headers for modification. + + Assert that the transformation kept them valid. + + """ + headers = Headers() + build_response(headers, key) + yield headers + check_response(headers, key) + + @contextlib.contextmanager + def assertInvalidResponseHeaders(self, exc_type, key="CSIRmL8dWYxeAdr/XpEHRw=="): + """ + Provide response headers for modification. + + Assert that the transformation made them invalid. + + """ + headers = Headers() + build_response(headers, key) + yield headers + assert issubclass(exc_type, InvalidHandshake) + with self.assertRaises(exc_type): + check_response(headers, key) + + def test_response_invalid_connection(self): + with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: + del headers["Connection"] + headers["Connection"] = "Downgrade" + + def test_response_missing_connection(self): + with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: + del headers["Connection"] + + def test_response_additional_connection(self): + with self.assertValidResponseHeaders() as headers: + headers["Connection"] = "close" + + def test_response_invalid_upgrade(self): + with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: + del headers["Upgrade"] + headers["Upgrade"] = "socketweb" + + def test_response_missing_upgrade(self): + with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: + del headers["Upgrade"] + + def test_response_additional_upgrade(self): + with self.assertInvalidResponseHeaders(InvalidUpgrade) as headers: + headers["Upgrade"] = "socketweb" + + def test_response_invalid_accept(self): + with self.assertInvalidResponseHeaders(InvalidHeaderValue) as headers: + del headers["Sec-WebSocket-Accept"] + other_key = "1Eq4UDEFQYg3YspNgqxv5g==" + headers["Sec-WebSocket-Accept"] = accept(other_key) + + def test_response_missing_accept(self): + with self.assertInvalidResponseHeaders(InvalidHeader) as headers: + del headers["Sec-WebSocket-Accept"] + + def test_response_additional_accept(self): + with self.assertInvalidResponseHeaders(InvalidHeader) as headers: + # This duplicates the Sec-WebSocket-Accept header. + headers["Sec-WebSocket-Accept"] = headers["Sec-WebSocket-Accept"] diff --git a/tests/test_http.py b/tests/test_http.py index b09247c3e..322650354 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,135 +1,2 @@ -import asyncio - -from websockets.exceptions import SecurityError -from websockets.http import * -from websockets.http import read_headers - -from .utils import AsyncioTestCase - - -class HTTPAsyncTests(AsyncioTestCase): - def setUp(self): - super().setUp() - self.stream = asyncio.StreamReader(loop=self.loop) - - async def test_read_request(self): - # Example from the protocol overview in RFC 6455 - self.stream.feed_data( - b"GET /chat HTTP/1.1\r\n" - b"Host: server.example.com\r\n" - b"Upgrade: websocket\r\n" - b"Connection: Upgrade\r\n" - b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" - b"Origin: http://example.com\r\n" - b"Sec-WebSocket-Protocol: chat, superchat\r\n" - b"Sec-WebSocket-Version: 13\r\n" - b"\r\n" - ) - path, headers = await read_request(self.stream) - self.assertEqual(path, "/chat") - self.assertEqual(headers["Upgrade"], "websocket") - - async def test_read_request_empty(self): - self.stream.feed_eof() - with self.assertRaisesRegex( - EOFError, "connection closed while reading HTTP request line" - ): - await read_request(self.stream) - - async def test_read_request_invalid_request_line(self): - self.stream.feed_data(b"GET /\r\n\r\n") - with self.assertRaisesRegex(ValueError, "invalid HTTP request line: GET /"): - await read_request(self.stream) - - async def test_read_request_unsupported_method(self): - self.stream.feed_data(b"OPTIONS * HTTP/1.1\r\n\r\n") - with self.assertRaisesRegex(ValueError, "unsupported HTTP method: OPTIONS"): - await read_request(self.stream) - - async def test_read_request_unsupported_version(self): - self.stream.feed_data(b"GET /chat HTTP/1.0\r\n\r\n") - with self.assertRaisesRegex(ValueError, "unsupported HTTP version: HTTP/1.0"): - await read_request(self.stream) - - async def test_read_request_invalid_header(self): - self.stream.feed_data(b"GET /chat HTTP/1.1\r\nOops\r\n") - with self.assertRaisesRegex(ValueError, "invalid HTTP header line: Oops"): - await read_request(self.stream) - - async def test_read_response(self): - # Example from the protocol overview in RFC 6455 - self.stream.feed_data( - b"HTTP/1.1 101 Switching Protocols\r\n" - b"Upgrade: websocket\r\n" - b"Connection: Upgrade\r\n" - b"Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n" - b"Sec-WebSocket-Protocol: chat\r\n" - b"\r\n" - ) - status_code, reason, headers = await read_response(self.stream) - self.assertEqual(status_code, 101) - self.assertEqual(reason, "Switching Protocols") - self.assertEqual(headers["Upgrade"], "websocket") - - async def test_read_response_empty(self): - self.stream.feed_eof() - with self.assertRaisesRegex( - EOFError, "connection closed while reading HTTP status line" - ): - await read_response(self.stream) - - async def test_read_request_invalid_status_line(self): - self.stream.feed_data(b"Hello!\r\n") - with self.assertRaisesRegex(ValueError, "invalid HTTP status line: Hello!"): - await read_response(self.stream) - - async def test_read_response_unsupported_version(self): - self.stream.feed_data(b"HTTP/1.0 400 Bad Request\r\n\r\n") - with self.assertRaisesRegex(ValueError, "unsupported HTTP version: HTTP/1.0"): - await read_response(self.stream) - - async def test_read_response_invalid_status(self): - self.stream.feed_data(b"HTTP/1.1 OMG WTF\r\n\r\n") - with self.assertRaisesRegex(ValueError, "invalid HTTP status code: OMG"): - await read_response(self.stream) - - async def test_read_response_unsupported_status(self): - self.stream.feed_data(b"HTTP/1.1 007 My name is Bond\r\n\r\n") - with self.assertRaisesRegex(ValueError, "unsupported HTTP status code: 007"): - await read_response(self.stream) - - async def test_read_response_invalid_reason(self): - self.stream.feed_data(b"HTTP/1.1 200 \x7f\r\n\r\n") - with self.assertRaisesRegex(ValueError, "invalid HTTP reason phrase: \\x7f"): - await read_response(self.stream) - - async def test_read_response_invalid_header(self): - self.stream.feed_data(b"HTTP/1.1 500 Internal Server Error\r\nOops\r\n") - with self.assertRaisesRegex(ValueError, "invalid HTTP header line: Oops"): - await read_response(self.stream) - - async def test_header_name(self): - self.stream.feed_data(b"foo bar: baz qux\r\n\r\n") - with self.assertRaises(ValueError): - await read_headers(self.stream) - - async def test_header_value(self): - self.stream.feed_data(b"foo: \x00\x00\x0f\r\n\r\n") - with self.assertRaises(ValueError): - await read_headers(self.stream) - - async def test_headers_limit(self): - self.stream.feed_data(b"foo: bar\r\n" * 257 + b"\r\n") - with self.assertRaises(SecurityError): - await read_headers(self.stream) - - async def test_line_limit(self): - # Header line contains 5 + 4090 + 2 = 4097 bytes. - self.stream.feed_data(b"foo: " + b"a" * 4090 + b"\r\n\r\n") - with self.assertRaises(SecurityError): - await read_headers(self.stream) - - async def test_line_ending(self): - self.stream.feed_data(b"foo: bar\n\n") - with self.assertRaises(EOFError): - await read_headers(self.stream) +# Check that the legacy http module imports without an exception. +from websockets.http import * # noqa diff --git a/tests/test_http_legacy.py b/tests/test_http_legacy.py new file mode 100644 index 000000000..3b43a6274 --- /dev/null +++ b/tests/test_http_legacy.py @@ -0,0 +1,135 @@ +import asyncio + +from websockets.exceptions import SecurityError +from websockets.http_legacy import * +from websockets.http_legacy import read_headers + +from .utils import AsyncioTestCase + + +class HTTPAsyncTests(AsyncioTestCase): + def setUp(self): + super().setUp() + self.stream = asyncio.StreamReader(loop=self.loop) + + async def test_read_request(self): + # Example from the protocol overview in RFC 6455 + self.stream.feed_data( + b"GET /chat HTTP/1.1\r\n" + b"Host: server.example.com\r\n" + b"Upgrade: websocket\r\n" + b"Connection: Upgrade\r\n" + b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + b"Origin: http://example.com\r\n" + b"Sec-WebSocket-Protocol: chat, superchat\r\n" + b"Sec-WebSocket-Version: 13\r\n" + b"\r\n" + ) + path, headers = await read_request(self.stream) + self.assertEqual(path, "/chat") + self.assertEqual(headers["Upgrade"], "websocket") + + async def test_read_request_empty(self): + self.stream.feed_eof() + with self.assertRaisesRegex( + EOFError, "connection closed while reading HTTP request line" + ): + await read_request(self.stream) + + async def test_read_request_invalid_request_line(self): + self.stream.feed_data(b"GET /\r\n\r\n") + with self.assertRaisesRegex(ValueError, "invalid HTTP request line: GET /"): + await read_request(self.stream) + + async def test_read_request_unsupported_method(self): + self.stream.feed_data(b"OPTIONS * HTTP/1.1\r\n\r\n") + with self.assertRaisesRegex(ValueError, "unsupported HTTP method: OPTIONS"): + await read_request(self.stream) + + async def test_read_request_unsupported_version(self): + self.stream.feed_data(b"GET /chat HTTP/1.0\r\n\r\n") + with self.assertRaisesRegex(ValueError, "unsupported HTTP version: HTTP/1.0"): + await read_request(self.stream) + + async def test_read_request_invalid_header(self): + self.stream.feed_data(b"GET /chat HTTP/1.1\r\nOops\r\n") + with self.assertRaisesRegex(ValueError, "invalid HTTP header line: Oops"): + await read_request(self.stream) + + async def test_read_response(self): + # Example from the protocol overview in RFC 6455 + self.stream.feed_data( + b"HTTP/1.1 101 Switching Protocols\r\n" + b"Upgrade: websocket\r\n" + b"Connection: Upgrade\r\n" + b"Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n" + b"Sec-WebSocket-Protocol: chat\r\n" + b"\r\n" + ) + status_code, reason, headers = await read_response(self.stream) + self.assertEqual(status_code, 101) + self.assertEqual(reason, "Switching Protocols") + self.assertEqual(headers["Upgrade"], "websocket") + + async def test_read_response_empty(self): + self.stream.feed_eof() + with self.assertRaisesRegex( + EOFError, "connection closed while reading HTTP status line" + ): + await read_response(self.stream) + + async def test_read_request_invalid_status_line(self): + self.stream.feed_data(b"Hello!\r\n") + with self.assertRaisesRegex(ValueError, "invalid HTTP status line: Hello!"): + await read_response(self.stream) + + async def test_read_response_unsupported_version(self): + self.stream.feed_data(b"HTTP/1.0 400 Bad Request\r\n\r\n") + with self.assertRaisesRegex(ValueError, "unsupported HTTP version: HTTP/1.0"): + await read_response(self.stream) + + async def test_read_response_invalid_status(self): + self.stream.feed_data(b"HTTP/1.1 OMG WTF\r\n\r\n") + with self.assertRaisesRegex(ValueError, "invalid HTTP status code: OMG"): + await read_response(self.stream) + + async def test_read_response_unsupported_status(self): + self.stream.feed_data(b"HTTP/1.1 007 My name is Bond\r\n\r\n") + with self.assertRaisesRegex(ValueError, "unsupported HTTP status code: 007"): + await read_response(self.stream) + + async def test_read_response_invalid_reason(self): + self.stream.feed_data(b"HTTP/1.1 200 \x7f\r\n\r\n") + with self.assertRaisesRegex(ValueError, "invalid HTTP reason phrase: \\x7f"): + await read_response(self.stream) + + async def test_read_response_invalid_header(self): + self.stream.feed_data(b"HTTP/1.1 500 Internal Server Error\r\nOops\r\n") + with self.assertRaisesRegex(ValueError, "invalid HTTP header line: Oops"): + await read_response(self.stream) + + async def test_header_name(self): + self.stream.feed_data(b"foo bar: baz qux\r\n\r\n") + with self.assertRaises(ValueError): + await read_headers(self.stream) + + async def test_header_value(self): + self.stream.feed_data(b"foo: \x00\x00\x0f\r\n\r\n") + with self.assertRaises(ValueError): + await read_headers(self.stream) + + async def test_headers_limit(self): + self.stream.feed_data(b"foo: bar\r\n" * 257 + b"\r\n") + with self.assertRaises(SecurityError): + await read_headers(self.stream) + + async def test_line_limit(self): + # Header line contains 5 + 4090 + 2 = 4097 bytes. + self.stream.feed_data(b"foo: " + b"a" * 4090 + b"\r\n\r\n") + with self.assertRaises(SecurityError): + await read_headers(self.stream) + + async def test_line_ending(self): + self.stream.feed_data(b"foo: bar\n\n") + with self.assertRaises(EOFError): + await read_headers(self.stream) From 1c99e5b9fabd3b431c5697a90193ef8e1cd17d58 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 6 Oct 2019 12:34:07 +0200 Subject: [PATCH 021/104] Move all type definitions to the typing module. --- src/websockets/handshake_legacy.py | 3 ++- src/websockets/headers.py | 14 +++++++++----- src/websockets/typing.py | 14 ++++++++++---- tests/test_typing.py | 1 + 4 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 tests/test_typing.py diff --git a/src/websockets/handshake_legacy.py b/src/websockets/handshake_legacy.py index 3fca45545..9683e8556 100644 --- a/src/websockets/handshake_legacy.py +++ b/src/websockets/handshake_legacy.py @@ -34,7 +34,8 @@ from .datastructures import Headers, MultipleValuesError from .exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade from .handshake import GUID -from .headers import ConnectionOption, UpgradeProtocol, parse_connection, parse_upgrade +from .headers import parse_connection, parse_upgrade +from .typing import ConnectionOption, UpgradeProtocol __all__ = ["build_request", "check_request", "build_response", "check_response"] diff --git a/src/websockets/headers.py b/src/websockets/headers.py index f33c94c04..256c66bb1 100644 --- a/src/websockets/headers.py +++ b/src/websockets/headers.py @@ -10,10 +10,17 @@ import base64 import binascii import re -from typing import Callable, List, NewType, Optional, Sequence, Tuple, TypeVar, cast +from typing import Callable, List, Optional, Sequence, Tuple, TypeVar, cast from .exceptions import InvalidHeaderFormat, InvalidHeaderValue -from .typing import ExtensionHeader, ExtensionName, ExtensionParameter, Subprotocol +from .typing import ( + ConnectionOption, + ExtensionHeader, + ExtensionName, + ExtensionParameter, + Subprotocol, + UpgradeProtocol, +) __all__ = [ @@ -31,9 +38,6 @@ T = TypeVar("T") -ConnectionOption = NewType("ConnectionOption", str) -UpgradeProtocol = NewType("UpgradeProtocol", str) - # To avoid a dependency on a parsing library, we implement manually the ABNF # described in https://tools.ietf.org/html/rfc6455#section-9.1 with the diff --git a/src/websockets/typing.py b/src/websockets/typing.py index a5062bc4b..ca66a8c54 100644 --- a/src/websockets/typing.py +++ b/src/websockets/typing.py @@ -28,7 +28,6 @@ ExtensionParameter = Tuple[str, Optional[str]] - ExtensionParameter__doc__ = """Parameter of a WebSocket extension""" try: ExtensionParameter.__doc__ = ExtensionParameter__doc__ @@ -37,8 +36,7 @@ ExtensionHeader = Tuple[ExtensionName, List[ExtensionParameter]] - -ExtensionHeader__doc__ = """Item parsed in a Sec-WebSocket-Extensions header""" +ExtensionHeader__doc__ = """Extension in a Sec-WebSocket-Extensions header""" try: ExtensionHeader.__doc__ = ExtensionHeader__doc__ except AttributeError: # pragma: no cover @@ -46,4 +44,12 @@ Subprotocol = NewType("Subprotocol", str) -Subprotocol.__doc__ = """Items parsed in a Sec-WebSocket-Protocol header""" +Subprotocol.__doc__ = """Subprotocol value in a Sec-WebSocket-Protocol header""" + + +ConnectionOption = NewType("ConnectionOption", str) +ConnectionOption.__doc__ = """Connection option in a Connection header""" + + +UpgradeProtocol = NewType("UpgradeProtocol", str) +UpgradeProtocol.__doc__ = """Upgrade protocol in an Upgrade header""" diff --git a/tests/test_typing.py b/tests/test_typing.py new file mode 100644 index 000000000..6eb1fe6c5 --- /dev/null +++ b/tests/test_typing.py @@ -0,0 +1 @@ +from websockets.typing import * # noqa From 80aea12a584b504f77e5a186c4c6b26444233297 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 25 Jan 2020 19:37:12 +0100 Subject: [PATCH 022/104] Add a StreamReader based on generator coroutines. --- src/websockets/streams.py | 115 ++++++++++++++++++++++++++++++ tests/test_streams.py | 146 ++++++++++++++++++++++++++++++++++++++ tests/utils.py | 18 +++++ 3 files changed, 279 insertions(+) create mode 100644 src/websockets/streams.py create mode 100644 tests/test_streams.py diff --git a/src/websockets/streams.py b/src/websockets/streams.py new file mode 100644 index 000000000..6f3163034 --- /dev/null +++ b/src/websockets/streams.py @@ -0,0 +1,115 @@ +from typing import Generator + + +class StreamReader: + """ + Generator-based stream reader. + + This class doesn't support concurrent calls to :meth:`read_line()`, + :meth:`read_exact()`, or :meth:`read_to_eof()`. Make sure calls are + serialized. + + """ + + def __init__(self) -> None: + self.buffer = bytearray() + self.eof = False + + def read_line(self) -> Generator[None, None, bytes]: + """ + Read a LF-terminated line from the stream. + + The return value includes the LF character. + + This is a generator-based coroutine. + + :raises EOFError: if the stream ends without a LF + + """ + n = 0 # number of bytes to read + p = 0 # number of bytes without a newline + while True: + n = self.buffer.find(b"\n", p) + 1 + if n > 0: + break + p = len(self.buffer) + if self.eof: + raise EOFError(f"stream ends after {p} bytes, before end of line") + yield + r = self.buffer[:n] + del self.buffer[:n] + return r + + def read_exact(self, n: int) -> Generator[None, None, bytes]: + """ + Read ``n`` bytes from the stream. + + This is a generator-based coroutine. + + :raises EOFError: if the stream ends in less than ``n`` bytes + + """ + assert n >= 0 + while len(self.buffer) < n: + if self.eof: + p = len(self.buffer) + raise EOFError(f"stream ends after {p} bytes, expected {n} bytes") + yield + r = self.buffer[:n] + del self.buffer[:n] + return r + + def read_to_eof(self) -> Generator[None, None, bytes]: + """ + Read all bytes from the stream. + + This is a generator-based coroutine. + + """ + while not self.eof: + yield + r = self.buffer[:] + del self.buffer[:] + return r + + def at_eof(self) -> Generator[None, None, bool]: + """ + Tell whether the stream has ended and all data was read. + + This is a generator-based coroutine. + + """ + while True: + if self.buffer: + return False + if self.eof: + return True + # When all data was read but the stream hasn't ended, we can't + # tell if until either feed_data() or feed_eof() is called. + yield + + def feed_data(self, data: bytes) -> None: + """ + Write ``data`` to the stream. + + :meth:`feed_data()` cannot be called after :meth:`feed_eof()`. + + :raises EOFError: if the stream has ended + + """ + if self.eof: + raise EOFError("stream ended") + self.buffer += data + + def feed_eof(self) -> None: + """ + End the stream. + + :meth:`feed_eof()` must be called at must once. + + :raises EOFError: if the stream has ended + + """ + if self.eof: + raise EOFError("stream ended") + self.eof = True diff --git a/tests/test_streams.py b/tests/test_streams.py new file mode 100644 index 000000000..566deb2db --- /dev/null +++ b/tests/test_streams.py @@ -0,0 +1,146 @@ +from websockets.streams import StreamReader + +from .utils import GeneratorTestCase + + +class StreamReaderTests(GeneratorTestCase): + def setUp(self): + self.reader = StreamReader() + + def test_read_line(self): + self.reader.feed_data(b"spam\neggs\n") + + gen = self.reader.read_line() + line = self.assertGeneratorReturns(gen) + self.assertEqual(line, b"spam\n") + + gen = self.reader.read_line() + line = self.assertGeneratorReturns(gen) + self.assertEqual(line, b"eggs\n") + + def test_read_line_need_more_data(self): + self.reader.feed_data(b"spa") + + gen = self.reader.read_line() + self.assertGeneratorRunning(gen) + self.reader.feed_data(b"m\neg") + line = self.assertGeneratorReturns(gen) + self.assertEqual(line, b"spam\n") + + gen = self.reader.read_line() + self.assertGeneratorRunning(gen) + self.reader.feed_data(b"gs\n") + line = self.assertGeneratorReturns(gen) + self.assertEqual(line, b"eggs\n") + + def test_read_line_not_enough_data(self): + self.reader.feed_data(b"spa") + self.reader.feed_eof() + + gen = self.reader.read_line() + with self.assertRaises(EOFError) as raised: + next(gen) + self.assertEqual( + str(raised.exception), "stream ends after 3 bytes, before end of line" + ) + + def test_read_exact(self): + self.reader.feed_data(b"spameggs") + + gen = self.reader.read_exact(4) + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"spam") + + gen = self.reader.read_exact(4) + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"eggs") + + def test_read_exact_need_more_data(self): + self.reader.feed_data(b"spa") + + gen = self.reader.read_exact(4) + self.assertGeneratorRunning(gen) + self.reader.feed_data(b"meg") + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"spam") + + gen = self.reader.read_exact(4) + self.assertGeneratorRunning(gen) + self.reader.feed_data(b"gs") + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"eggs") + + def test_read_exact_not_enough_data(self): + self.reader.feed_data(b"spa") + self.reader.feed_eof() + + gen = self.reader.read_exact(4) + with self.assertRaises(EOFError) as raised: + next(gen) + self.assertEqual( + str(raised.exception), "stream ends after 3 bytes, expected 4 bytes" + ) + + def test_read_to_eof(self): + gen = self.reader.read_to_eof() + + self.reader.feed_data(b"spam") + self.assertGeneratorRunning(gen) + + self.reader.feed_eof() + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"spam") + + def test_read_to_eof_at_eof(self): + self.reader.feed_eof() + + gen = self.reader.read_to_eof() + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"") + + def test_at_eof_after_feed_data(self): + gen = self.reader.at_eof() + self.assertGeneratorRunning(gen) + self.reader.feed_data(b"spam") + eof = self.assertGeneratorReturns(gen) + self.assertFalse(eof) + + def test_at_eof_after_feed_eof(self): + gen = self.reader.at_eof() + self.assertGeneratorRunning(gen) + self.reader.feed_eof() + eof = self.assertGeneratorReturns(gen) + self.assertTrue(eof) + + def test_feed_data_after_feed_data(self): + self.reader.feed_data(b"spam") + self.reader.feed_data(b"eggs") + + gen = self.reader.read_exact(8) + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"spameggs") + gen = self.reader.at_eof() + self.assertGeneratorRunning(gen) + + def test_feed_eof_after_feed_data(self): + self.reader.feed_data(b"spam") + self.reader.feed_eof() + + gen = self.reader.read_exact(4) + data = self.assertGeneratorReturns(gen) + self.assertEqual(data, b"spam") + gen = self.reader.at_eof() + eof = self.assertGeneratorReturns(gen) + self.assertTrue(eof) + + def test_feed_data_after_feed_eof(self): + self.reader.feed_eof() + with self.assertRaises(EOFError) as raised: + self.reader.feed_data(b"spam") + self.assertEqual(str(raised.exception), "stream ended") + + def test_feed_eof_after_feed_eof(self): + self.reader.feed_eof() + with self.assertRaises(EOFError) as raised: + self.reader.feed_eof() + self.assertEqual(str(raised.exception), "stream ended") diff --git a/tests/utils.py b/tests/utils.py index 983a91edf..bbffa8649 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -7,6 +7,24 @@ import unittest +class GeneratorTestCase(unittest.TestCase): + def assertGeneratorRunning(self, gen): + """ + Check that a generator-based coroutine hasn't completed yet. + + """ + next(gen) + + def assertGeneratorReturns(self, gen): + """ + Check that a generator-based coroutine completes and return its value. + + """ + with self.assertRaises(StopIteration) as raised: + next(gen) + return raised.exception.value + + class AsyncioTestCase(unittest.TestCase): """ Base class for tests that sets up an isolated event loop for each test. From 624b9d20061c78df81f659af2c87557c764ebb19 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 6 Oct 2019 21:27:06 +0200 Subject: [PATCH 023/104] Add a sans-I/O compatible framing implementation. --- docs/changelog.rst | 2 + src/websockets/extensions/base.py | 2 +- .../extensions/permessage_deflate.py | 2 +- src/websockets/frames.py | 322 ++++++++++++++++++ src/websockets/framing.py | 233 +------------ src/websockets/protocol.py | 19 +- tests/__init__.py | 10 + tests/extensions/test_permessage_deflate.py | 2 +- tests/test_frames.py | 232 +++++++++++++ tests/test_framing.py | 103 +----- tests/test_protocol.py | 11 +- 11 files changed, 624 insertions(+), 314 deletions(-) create mode 100644 src/websockets/frames.py create mode 100644 tests/test_frames.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 3cda4919f..68ec6f80c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,8 @@ Changelog * :mod:`websockets.http` is deprecated. + * :mod:`websocket.framing` is deprecated. + Aliases provide backwards compatibility for all previously public APIs. 8.1 diff --git a/src/websockets/extensions/base.py b/src/websockets/extensions/base.py index aa52a7adb..cfc090799 100644 --- a/src/websockets/extensions/base.py +++ b/src/websockets/extensions/base.py @@ -10,7 +10,7 @@ from typing import List, Optional, Sequence, Tuple -from ..framing import Frame +from ..frames import Frame from ..typing import ExtensionName, ExtensionParameter diff --git a/src/websockets/extensions/permessage_deflate.py b/src/websockets/extensions/permessage_deflate.py index e38d9edab..f1adf8bb6 100644 --- a/src/websockets/extensions/permessage_deflate.py +++ b/src/websockets/extensions/permessage_deflate.py @@ -14,7 +14,7 @@ NegotiationError, PayloadTooBig, ) -from ..framing import CTRL_OPCODES, OP_CONT, Frame +from ..frames import CTRL_OPCODES, OP_CONT, Frame from ..typing import ExtensionName, ExtensionParameter from .base import ClientExtensionFactory, Extension, ServerExtensionFactory diff --git a/src/websockets/frames.py b/src/websockets/frames.py new file mode 100644 index 000000000..5ed8e483f --- /dev/null +++ b/src/websockets/frames.py @@ -0,0 +1,322 @@ +""" +Parse and serialize WebSocket frames. + +""" + +import io +import random +import struct +from typing import Callable, Generator, NamedTuple, Optional, Sequence, Tuple + +from .exceptions import PayloadTooBig, ProtocolError +from .typing import Data + + +try: + from .speedups import apply_mask +except ImportError: # pragma: no cover + from .utils import apply_mask + + +__all__ = [ + "DATA_OPCODES", + "CTRL_OPCODES", + "OP_CONT", + "OP_TEXT", + "OP_BINARY", + "OP_CLOSE", + "OP_PING", + "OP_PONG", + "Frame", + "prepare_data", + "prepare_ctrl", + "parse_close", + "serialize_close", +] + +DATA_OPCODES = OP_CONT, OP_TEXT, OP_BINARY = 0x00, 0x01, 0x02 +CTRL_OPCODES = OP_CLOSE, OP_PING, OP_PONG = 0x08, 0x09, 0x0A + +# Close code that are allowed in a close frame. +# Using a list optimizes `code in EXTERNAL_CLOSE_CODES`. +EXTERNAL_CLOSE_CODES = [1000, 1001, 1002, 1003, 1007, 1008, 1009, 1010, 1011] + + +# Consider converting to a dataclass when dropping support for Python < 3.7. + + +class Frame(NamedTuple): + """ + WebSocket frame. + + :param bool fin: FIN bit + :param bool rsv1: RSV1 bit + :param bool rsv2: RSV2 bit + :param bool rsv3: RSV3 bit + :param int opcode: opcode + :param bytes data: payload data + + Only these fields are needed. The MASK bit, payload length and masking-key + are handled on the fly by :func:`parse_frame` and :meth:`serialize_frame`. + + """ + + fin: bool + opcode: int + data: bytes + rsv1: bool = False + rsv2: bool = False + rsv3: bool = False + + @classmethod + def parse( + cls, + read_exact: Callable[[int], Generator[None, None, bytes]], + *, + mask: bool, + max_size: Optional[int] = None, + extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + ) -> Generator[None, None, "Frame"]: + """ + Read a WebSocket frame. + + :param read_exact: generator-based coroutine that reads the requested + number of bytes or raises an exception if there isn't enough data + :param mask: whether the frame should be masked i.e. whether the read + happens on the server side + :param max_size: maximum payload size in bytes + :param extensions: list of classes with a ``decode()`` method that + transforms the frame and return a new frame; extensions are applied + in reverse order + :raises ~websockets.exceptions.PayloadTooBig: if the frame exceeds + ``max_size`` + :raises ~websockets.exceptions.ProtocolError: if the frame + contains incorrect values + + """ + # Read the header. + data = yield from read_exact(2) + head1, head2 = struct.unpack("!BB", data) + + # While not Pythonic, this is marginally faster than calling bool(). + fin = True if head1 & 0b10000000 else False + rsv1 = True if head1 & 0b01000000 else False + rsv2 = True if head1 & 0b00100000 else False + rsv3 = True if head1 & 0b00010000 else False + opcode = head1 & 0b00001111 + + if (True if head2 & 0b10000000 else False) != mask: + raise ProtocolError("incorrect masking") + + length = head2 & 0b01111111 + if length == 126: + data = yield from read_exact(2) + (length,) = struct.unpack("!H", data) + elif length == 127: + data = yield from read_exact(8) + (length,) = struct.unpack("!Q", data) + if max_size is not None and length > max_size: + raise PayloadTooBig( + f"payload length exceeds size limit ({length} > {max_size} bytes)" + ) + if mask: + mask_bits = yield from read_exact(4) + + # Read the data. + data = yield from read_exact(length) + if mask: + data = apply_mask(data, mask_bits) + + frame = cls(fin, opcode, data, rsv1, rsv2, rsv3) + + if extensions is None: + extensions = [] + for extension in reversed(extensions): + frame = extension.decode(frame, max_size=max_size) + + frame.check() + + return frame + + def serialize( + self, + *, + mask: bool, + extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + ) -> bytes: + """ + Write a WebSocket frame. + + :param frame: frame to write + :param mask: whether the frame should be masked i.e. whether the write + happens on the client side + :param extensions: list of classes with an ``encode()`` method that + transform the frame and return a new frame; extensions are applied + in order + :raises ~websockets.exceptions.ProtocolError: if the frame + contains incorrect values + + """ + self.check() + + if extensions is None: + extensions = [] + for extension in extensions: + self = extension.encode(self) + + output = io.BytesIO() + + # Prepare the header. + head1 = ( + (0b10000000 if self.fin else 0) + | (0b01000000 if self.rsv1 else 0) + | (0b00100000 if self.rsv2 else 0) + | (0b00010000 if self.rsv3 else 0) + | self.opcode + ) + + head2 = 0b10000000 if mask else 0 + + length = len(self.data) + if length < 126: + output.write(struct.pack("!BB", head1, head2 | length)) + elif length < 65536: + output.write(struct.pack("!BBH", head1, head2 | 126, length)) + else: + output.write(struct.pack("!BBQ", head1, head2 | 127, length)) + + if mask: + mask_bits = struct.pack("!I", random.getrandbits(32)) + output.write(mask_bits) + + # Prepare the data. + if mask: + data = apply_mask(self.data, mask_bits) + else: + data = self.data + output.write(data) + + return output.getvalue() + + def check(self) -> None: + """ + Check that reserved bits and opcode have acceptable values. + + :raises ~websockets.exceptions.ProtocolError: if a reserved + bit or the opcode is invalid + + """ + if self.rsv1 or self.rsv2 or self.rsv3: + raise ProtocolError("reserved bits must be 0") + + if self.opcode in DATA_OPCODES: + return + elif self.opcode in CTRL_OPCODES: + if len(self.data) > 125: + raise ProtocolError("control frame too long") + if not self.fin: + raise ProtocolError("fragmented control frame") + else: + raise ProtocolError(f"invalid opcode: {self.opcode}") + + +def prepare_data(data: Data) -> Tuple[int, bytes]: + """ + Convert a string or byte-like object to an opcode and a bytes-like object. + + This function is designed for data frames. + + If ``data`` is a :class:`str`, return ``OP_TEXT`` and a :class:`bytes` + object encoding ``data`` in UTF-8. + + If ``data`` is a bytes-like object, return ``OP_BINARY`` and a bytes-like + object. + + :raises TypeError: if ``data`` doesn't have a supported type + + """ + if isinstance(data, str): + return OP_TEXT, data.encode("utf-8") + elif isinstance(data, (bytes, bytearray)): + return OP_BINARY, data + elif isinstance(data, memoryview): + if data.c_contiguous: + return OP_BINARY, data + else: + return OP_BINARY, data.tobytes() + else: + raise TypeError("data must be bytes-like or str") + + +def prepare_ctrl(data: Data) -> bytes: + """ + Convert a string or byte-like object to bytes. + + This function is designed for ping and pong frames. + + If ``data`` is a :class:`str`, return a :class:`bytes` object encoding + ``data`` in UTF-8. + + If ``data`` is a bytes-like object, return a :class:`bytes` object. + + :raises TypeError: if ``data`` doesn't have a supported type + + """ + if isinstance(data, str): + return data.encode("utf-8") + elif isinstance(data, (bytes, bytearray)): + return bytes(data) + elif isinstance(data, memoryview): + return data.tobytes() + else: + raise TypeError("data must be bytes-like or str") + + +def parse_close(data: bytes) -> Tuple[int, str]: + """ + Parse the payload from a close frame. + + Return ``(code, reason)``. + + :raises ~websockets.exceptions.ProtocolError: if data is ill-formed + :raises UnicodeDecodeError: if the reason isn't valid UTF-8 + + """ + length = len(data) + if length >= 2: + (code,) = struct.unpack("!H", data[:2]) + check_close(code) + reason = data[2:].decode("utf-8") + return code, reason + elif length == 0: + return 1005, "" + else: + assert length == 1 + raise ProtocolError("close frame too short") + + +def serialize_close(code: int, reason: str) -> bytes: + """ + Serialize the payload for a close frame. + + This is the reverse of :func:`parse_close`. + + """ + check_close(code) + return struct.pack("!H", code) + reason.encode("utf-8") + + +def check_close(code: int) -> None: + """ + Check that the close code has an acceptable value for a close frame. + + :raises ~websockets.exceptions.ProtocolError: if the close code + is invalid + + """ + if not (code in EXTERNAL_CLOSE_CODES or 3000 <= code < 5000): + raise ProtocolError("invalid status code") + + +# at the bottom to allow circular import, because Extension depends on Frame +import websockets.extensions.base # isort:skip # noqa diff --git a/src/websockets/framing.py b/src/websockets/framing.py index 26e58cdbf..221afad6f 100644 --- a/src/websockets/framing.py +++ b/src/websockets/framing.py @@ -10,13 +10,12 @@ """ -import io -import random import struct -from typing import Any, Awaitable, Callable, NamedTuple, Optional, Sequence, Tuple +import warnings +from typing import Any, Awaitable, Callable, Optional, Sequence from .exceptions import PayloadTooBig, ProtocolError -from .typing import Data +from .frames import Frame as NewFrame try: @@ -25,56 +24,10 @@ from .utils import apply_mask -__all__ = [ - "DATA_OPCODES", - "CTRL_OPCODES", - "OP_CONT", - "OP_TEXT", - "OP_BINARY", - "OP_CLOSE", - "OP_PING", - "OP_PONG", - "Frame", - "prepare_data", - "encode_data", - "parse_close", - "serialize_close", -] +warnings.warn("websockets.framing is deprecated", DeprecationWarning) -DATA_OPCODES = OP_CONT, OP_TEXT, OP_BINARY = 0x00, 0x01, 0x02 -CTRL_OPCODES = OP_CLOSE, OP_PING, OP_PONG = 0x08, 0x09, 0x0A - -# Close code that are allowed in a close frame. -# Using a list optimizes `code in EXTERNAL_CLOSE_CODES`. -EXTERNAL_CLOSE_CODES = [1000, 1001, 1002, 1003, 1007, 1008, 1009, 1010, 1011] - - -# Consider converting to a dataclass when dropping support for Python < 3.7. - - -class Frame(NamedTuple): - """ - WebSocket frame. - - :param bool fin: FIN bit - :param bool rsv1: RSV1 bit - :param bool rsv2: RSV2 bit - :param bool rsv3: RSV3 bit - :param int opcode: opcode - :param bytes data: payload data - - Only these fields are needed. The MASK bit, payload length and masking-key - are handled on the fly by :meth:`read` and :meth:`write`. - - """ - - fin: bool - opcode: int - data: bytes - rsv1: bool = False - rsv2: bool = False - rsv3: bool = False +class Frame(NewFrame): @classmethod async def read( cls, @@ -101,6 +54,7 @@ async def read( contains incorrect values """ + # Read the header. data = await reader(2) head1, head2 = struct.unpack("!BB", data) @@ -139,14 +93,14 @@ async def read( if extensions is None: extensions = [] for extension in reversed(extensions): - frame = extension.decode(frame, max_size=max_size) + frame = cls(*extension.decode(frame, max_size=max_size)) frame.check() return frame def write( - frame, + self, write: Callable[[bytes], Any], *, mask: bool, @@ -166,176 +120,17 @@ def write( contains incorrect values """ - # The first parameter is called `frame` rather than `self`, - # but it's the instance of class to which this method is bound. - - frame.check() - - if extensions is None: - extensions = [] - for extension in extensions: - frame = extension.encode(frame) - - output = io.BytesIO() - - # Prepare the header. - head1 = ( - (0b10000000 if frame.fin else 0) - | (0b01000000 if frame.rsv1 else 0) - | (0b00100000 if frame.rsv2 else 0) - | (0b00010000 if frame.rsv3 else 0) - | frame.opcode - ) - - head2 = 0b10000000 if mask else 0 - - length = len(frame.data) - if length < 126: - output.write(struct.pack("!BB", head1, head2 | length)) - elif length < 65536: - output.write(struct.pack("!BBH", head1, head2 | 126, length)) - else: - output.write(struct.pack("!BBQ", head1, head2 | 127, length)) - - if mask: - mask_bits = struct.pack("!I", random.getrandbits(32)) - output.write(mask_bits) - - # Prepare the data. - if mask: - data = apply_mask(frame.data, mask_bits) - else: - data = frame.data - output.write(data) - - # Send the frame. - # The frame is written in a single call to write in order to prevent # TCP fragmentation. See #68 for details. This also makes it safe to # send frames concurrently from multiple coroutines. - write(output.getvalue()) - - def check(frame) -> None: - """ - Check that reserved bits and opcode have acceptable values. - - :raises ~websockets.exceptions.ProtocolError: if a reserved - bit or the opcode is invalid - - """ - # The first parameter is called `frame` rather than `self`, - # but it's the instance of class to which this method is bound. - - if frame.rsv1 or frame.rsv2 or frame.rsv3: - raise ProtocolError("reserved bits must be 0") - - if frame.opcode in DATA_OPCODES: - return - elif frame.opcode in CTRL_OPCODES: - if len(frame.data) > 125: - raise ProtocolError("control frame too long") - if not frame.fin: - raise ProtocolError("fragmented control frame") - else: - raise ProtocolError(f"invalid opcode: {frame.opcode}") - - -def prepare_data(data: Data) -> Tuple[int, bytes]: - """ - Convert a string or byte-like object to an opcode and a bytes-like object. - - This function is designed for data frames. - - If ``data`` is a :class:`str`, return ``OP_TEXT`` and a :class:`bytes` - object encoding ``data`` in UTF-8. - - If ``data`` is a bytes-like object, return ``OP_BINARY`` and a bytes-like - object. - - :raises TypeError: if ``data`` doesn't have a supported type - - """ - if isinstance(data, str): - return OP_TEXT, data.encode("utf-8") - elif isinstance(data, (bytes, bytearray)): - return OP_BINARY, data - elif isinstance(data, memoryview): - if data.c_contiguous: - return OP_BINARY, data - else: - return OP_BINARY, data.tobytes() - else: - raise TypeError("data must be bytes-like or str") - - -def encode_data(data: Data) -> bytes: - """ - Convert a string or byte-like object to bytes. - - This function is designed for ping and pong frames. - - If ``data`` is a :class:`str`, return a :class:`bytes` object encoding - ``data`` in UTF-8. - - If ``data`` is a bytes-like object, return a :class:`bytes` object. - - :raises TypeError: if ``data`` doesn't have a supported type - - """ - if isinstance(data, str): - return data.encode("utf-8") - elif isinstance(data, (bytes, bytearray)): - return bytes(data) - elif isinstance(data, memoryview): - return data.tobytes() - else: - raise TypeError("data must be bytes-like or str") - - -def parse_close(data: bytes) -> Tuple[int, str]: - """ - Parse the payload from a close frame. - - Return ``(code, reason)``. - - :raises ~websockets.exceptions.ProtocolError: if data is ill-formed - :raises UnicodeDecodeError: if the reason isn't valid UTF-8 - - """ - length = len(data) - if length >= 2: - (code,) = struct.unpack("!H", data[:2]) - check_close(code) - reason = data[2:].decode("utf-8") - return code, reason - elif length == 0: - return 1005, "" - else: - assert length == 1 - raise ProtocolError("close frame too short") - - -def serialize_close(code: int, reason: str) -> bytes: - """ - Serialize the payload for a close frame. - - This is the reverse of :func:`parse_close`. - - """ - check_close(code) - return struct.pack("!H", code) + reason.encode("utf-8") - - -def check_close(code: int) -> None: - """ - Check that the close code has an acceptable value for a close frame. + write(self.serialize(mask=mask, extensions=extensions)) - :raises ~websockets.exceptions.ProtocolError: if the close code - is invalid - """ - if not (code in EXTERNAL_CLOSE_CODES or 3000 <= code < 5000): - raise ProtocolError("invalid status code") +# Backwards compatibility with previously documented public APIs +from .frames import parse_close # isort:skip # noqa +from .frames import prepare_ctrl as encode_data # isort:skip # noqa +from .frames import prepare_data # isort:skip # noqa +from .frames import serialize_close # isort:skip # noqa # at the bottom to allow circular import, because Extension depends on Frame diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index cc4416ba8..748c1ae66 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -40,8 +40,19 @@ ProtocolError, ) from .extensions.base import Extension -from .framing import * -from .handshake_legacy import * +from .frames import ( + OP_BINARY, + OP_CLOSE, + OP_CONT, + OP_PING, + OP_PONG, + OP_TEXT, + parse_close, + prepare_ctrl, + prepare_data, + serialize_close, +) +from .framing import Frame from .typing import Data @@ -732,7 +743,7 @@ async def ping(self, data: Optional[Data] = None) -> Awaitable[None]: await self.ensure_open() if data is not None: - data = encode_data(data) + data = prepare_ctrl(data) # Protect against duplicates if a payload is explicitly set. if data in self.pings: @@ -763,7 +774,7 @@ async def pong(self, data: Data = b"") -> None: """ await self.ensure_open() - data = encode_data(data) + data = prepare_ctrl(data) await self.write_frame(True, OP_PONG, data) diff --git a/tests/__init__.py b/tests/__init__.py index dd78609f5..76c869f50 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,15 @@ import logging +import warnings # Avoid displaying stack traces at the ERROR logging level. logging.basicConfig(level=logging.CRITICAL) + + +# Ignore deprecation warnings while refactoring is in progress +warnings.filterwarnings( + action="ignore", + message=r"websockets\.framing is deprecated", + category=DeprecationWarning, + module="websockets.framing", +) diff --git a/tests/extensions/test_permessage_deflate.py b/tests/extensions/test_permessage_deflate.py index 0ec49c6c0..e1193e672 100644 --- a/tests/extensions/test_permessage_deflate.py +++ b/tests/extensions/test_permessage_deflate.py @@ -9,7 +9,7 @@ PayloadTooBig, ) from websockets.extensions.permessage_deflate import * -from websockets.framing import ( +from websockets.frames import ( OP_BINARY, OP_CLOSE, OP_CONT, diff --git a/tests/test_frames.py b/tests/test_frames.py new file mode 100644 index 000000000..39d4055a8 --- /dev/null +++ b/tests/test_frames.py @@ -0,0 +1,232 @@ +import codecs +import struct +import unittest +import unittest.mock + +from websockets.exceptions import PayloadTooBig, ProtocolError +from websockets.frames import * +from websockets.streams import StreamReader + +from .utils import GeneratorTestCase + + +class FrameTests(GeneratorTestCase): + def parse(self, data, mask=False, max_size=None, extensions=None): + reader = StreamReader() + reader.feed_data(data) + reader.feed_eof() + parser = Frame.parse( + reader.read_exact, mask=mask, max_size=max_size, extensions=extensions, + ) + return self.assertGeneratorReturns(parser) + + def round_trip(self, data, frame, mask=False, extensions=None): + parsed = self.parse(data, mask=mask, extensions=extensions) + self.assertEqual(parsed, frame) + + # Make masking deterministic by reusing the same "random" mask. + # This has an effect only when mask is True. + randbits = struct.unpack("!I", data[2:6])[0] if mask else 0 + with unittest.mock.patch("random.getrandbits", return_value=randbits): + serialized = parsed.serialize(mask=mask, extensions=extensions) + self.assertEqual(serialized, data) + + def test_text(self): + self.round_trip(b"\x81\x04Spam", Frame(True, OP_TEXT, b"Spam")) + + def test_text_masked(self): + self.round_trip( + b"\x81\x84\x5b\xfb\xe1\xa8\x08\x8b\x80\xc5", + Frame(True, OP_TEXT, b"Spam"), + mask=True, + ) + + def test_binary(self): + self.round_trip(b"\x82\x04Eggs", Frame(True, OP_BINARY, b"Eggs")) + + def test_binary_masked(self): + self.round_trip( + b"\x82\x84\x53\xcd\xe2\x89\x16\xaa\x85\xfa", + Frame(True, OP_BINARY, b"Eggs"), + mask=True, + ) + + def test_non_ascii_text(self): + self.round_trip( + b"\x81\x05caf\xc3\xa9", Frame(True, OP_TEXT, "café".encode("utf-8")) + ) + + def test_non_ascii_text_masked(self): + self.round_trip( + b"\x81\x85\x64\xbe\xee\x7e\x07\xdf\x88\xbd\xcd", + Frame(True, OP_TEXT, "café".encode("utf-8")), + mask=True, + ) + + def test_close(self): + self.round_trip(b"\x88\x00", Frame(True, OP_CLOSE, b"")) + + def test_ping(self): + self.round_trip(b"\x89\x04ping", Frame(True, OP_PING, b"ping")) + + def test_pong(self): + self.round_trip(b"\x8a\x04pong", Frame(True, OP_PONG, b"pong")) + + def test_long(self): + self.round_trip( + b"\x82\x7e\x00\x7e" + 126 * b"a", Frame(True, OP_BINARY, 126 * b"a") + ) + + def test_very_long(self): + self.round_trip( + b"\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x00" + 65536 * b"a", + Frame(True, OP_BINARY, 65536 * b"a"), + ) + + def test_payload_too_big(self): + with self.assertRaises(PayloadTooBig): + self.parse(b"\x82\x7e\x04\x01" + 1025 * b"a", max_size=1024) + + def test_bad_reserved_bits(self): + for data in [b"\xc0\x00", b"\xa0\x00", b"\x90\x00"]: + with self.subTest(data=data): + with self.assertRaises(ProtocolError): + self.parse(data) + + def test_good_opcode(self): + for opcode in list(range(0x00, 0x03)) + list(range(0x08, 0x0B)): + data = bytes([0x80 | opcode, 0]) + with self.subTest(data=data): + self.parse(data) # does not raise an exception + + def test_bad_opcode(self): + for opcode in list(range(0x03, 0x08)) + list(range(0x0B, 0x10)): + data = bytes([0x80 | opcode, 0]) + with self.subTest(data=data): + with self.assertRaises(ProtocolError): + self.parse(data) + + def test_mask_flag(self): + # Mask flag correctly set. + self.parse(b"\x80\x80\x00\x00\x00\x00", mask=True) + # Mask flag incorrectly unset. + with self.assertRaises(ProtocolError): + self.parse(b"\x80\x80\x00\x00\x00\x00") + # Mask flag correctly unset. + self.parse(b"\x80\x00") + # Mask flag incorrectly set. + with self.assertRaises(ProtocolError): + self.parse(b"\x80\x00", mask=True) + + def test_control_frame_max_length(self): + # At maximum allowed length. + self.parse(b"\x88\x7e\x00\x7d" + 125 * b"a") + # Above maximum allowed length. + with self.assertRaises(ProtocolError): + self.parse(b"\x88\x7e\x00\x7e" + 126 * b"a") + + def test_fragmented_control_frame(self): + # Fin bit correctly set. + self.parse(b"\x88\x00") + # Fin bit incorrectly unset. + with self.assertRaises(ProtocolError): + self.parse(b"\x08\x00") + + def test_extensions(self): + class Rot13: + @staticmethod + def encode(frame): + assert frame.opcode == OP_TEXT + text = frame.data.decode() + data = codecs.encode(text, "rot13").encode() + return frame._replace(data=data) + + # This extensions is symmetrical. + @staticmethod + def decode(frame, *, max_size=None): + return Rot13.encode(frame) + + self.round_trip( + b"\x81\x05uryyb", Frame(True, OP_TEXT, b"hello"), extensions=[Rot13()] + ) + + +class PrepareDataTests(unittest.TestCase): + def test_prepare_data_str(self): + self.assertEqual(prepare_data("café"), (OP_TEXT, b"caf\xc3\xa9")) + + def test_prepare_data_bytes(self): + self.assertEqual(prepare_data(b"tea"), (OP_BINARY, b"tea")) + + def test_prepare_data_bytearray(self): + self.assertEqual( + prepare_data(bytearray(b"tea")), (OP_BINARY, bytearray(b"tea")) + ) + + def test_prepare_data_memoryview(self): + self.assertEqual( + prepare_data(memoryview(b"tea")), (OP_BINARY, memoryview(b"tea")) + ) + + def test_prepare_data_non_contiguous_memoryview(self): + self.assertEqual(prepare_data(memoryview(b"tteeaa")[::2]), (OP_BINARY, b"tea")) + + def test_prepare_data_list(self): + with self.assertRaises(TypeError): + prepare_data([]) + + def test_prepare_data_none(self): + with self.assertRaises(TypeError): + prepare_data(None) + + +class PrepareCtrlTests(unittest.TestCase): + def test_prepare_ctrl_str(self): + self.assertEqual(prepare_ctrl("café"), b"caf\xc3\xa9") + + def test_prepare_ctrl_bytes(self): + self.assertEqual(prepare_ctrl(b"tea"), b"tea") + + def test_prepare_ctrl_bytearray(self): + self.assertEqual(prepare_ctrl(bytearray(b"tea")), b"tea") + + def test_prepare_ctrl_memoryview(self): + self.assertEqual(prepare_ctrl(memoryview(b"tea")), b"tea") + + def test_prepare_ctrl_non_contiguous_memoryview(self): + self.assertEqual(prepare_ctrl(memoryview(b"tteeaa")[::2]), b"tea") + + def test_prepare_ctrl_list(self): + with self.assertRaises(TypeError): + prepare_ctrl([]) + + def test_prepare_ctrl_none(self): + with self.assertRaises(TypeError): + prepare_ctrl(None) + + +class ParseAndSerializeCloseTests(unittest.TestCase): + def round_trip(self, data, code, reason): + parsed = parse_close(data) + self.assertEqual(parsed, (code, reason)) + serialized = serialize_close(code, reason) + self.assertEqual(serialized, data) + + def test_parse_close_and_serialize_close(self): + self.round_trip(b"\x03\xe8", 1000, "") + self.round_trip(b"\x03\xe8OK", 1000, "OK") + + def test_parse_close_empty(self): + self.assertEqual(parse_close(b""), (1005, "")) + + def test_parse_close_errors(self): + with self.assertRaises(ProtocolError): + parse_close(b"\x03") + with self.assertRaises(ProtocolError): + parse_close(b"\x03\xe7") + with self.assertRaises(UnicodeDecodeError): + parse_close(b"\x03\xe8\xff\xff") + + def test_serialize_close_errors(self): + with self.assertRaises(ProtocolError): + serialize_close(999, "") diff --git a/tests/test_framing.py b/tests/test_framing.py index 5def415d2..231cbf718 100644 --- a/tests/test_framing.py +++ b/tests/test_framing.py @@ -2,8 +2,10 @@ import codecs import unittest import unittest.mock +import warnings from websockets.exceptions import PayloadTooBig, ProtocolError +from websockets.frames import OP_BINARY, OP_CLOSE, OP_PING, OP_PONG, OP_TEXT from websockets.framing import * from .utils import AsyncioTestCase @@ -11,24 +13,26 @@ class FramingTests(AsyncioTestCase): def decode(self, message, mask=False, max_size=None, extensions=None): - self.stream = asyncio.StreamReader(loop=self.loop) - self.stream.feed_data(message) - self.stream.feed_eof() - frame = self.loop.run_until_complete( - Frame.read( - self.stream.readexactly, - mask=mask, - max_size=max_size, - extensions=extensions, + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(message) + stream.feed_eof() + with warnings.catch_warnings(record=True): + frame = self.loop.run_until_complete( + Frame.read( + stream.readexactly, + mask=mask, + max_size=max_size, + extensions=extensions, + ) ) - ) # Make sure all the data was consumed. - self.assertTrue(self.stream.at_eof()) + self.assertTrue(stream.at_eof()) return frame def encode(self, frame, mask=False, extensions=None): write = unittest.mock.Mock() - frame.write(write, mask=mask, extensions=extensions) + with warnings.catch_warnings(record=True): + frame.write(write, mask=mask, extensions=extensions) # Ensure the entire frame is sent with a single call to write(). # Multiple calls cause TCP fragmentation and degrade performance. self.assertEqual(write.call_count, 1) @@ -47,12 +51,6 @@ def round_trip(self, message, expected, mask=False, extensions=None): else: # deterministic encoding self.assertEqual(encoded, message) - def round_trip_close(self, data, code, reason): - parsed = parse_close(data) - self.assertEqual(parsed, (code, reason)) - serialized = serialize_close(code, reason) - self.assertEqual(serialized, data) - def test_text(self): self.round_trip(b"\x81\x04Spam", Frame(True, OP_TEXT, b"Spam")) @@ -147,56 +145,6 @@ def test_control_frame_max_length(self): with self.assertRaises(ProtocolError): self.decode(b"\x88\x7e\x00\x7e" + 126 * b"a") - def test_prepare_data_str(self): - self.assertEqual(prepare_data("café"), (OP_TEXT, b"caf\xc3\xa9")) - - def test_prepare_data_bytes(self): - self.assertEqual(prepare_data(b"tea"), (OP_BINARY, b"tea")) - - def test_prepare_data_bytearray(self): - self.assertEqual( - prepare_data(bytearray(b"tea")), (OP_BINARY, bytearray(b"tea")) - ) - - def test_prepare_data_memoryview(self): - self.assertEqual( - prepare_data(memoryview(b"tea")), (OP_BINARY, memoryview(b"tea")) - ) - - def test_prepare_data_non_contiguous_memoryview(self): - self.assertEqual(prepare_data(memoryview(b"tteeaa")[::2]), (OP_BINARY, b"tea")) - - def test_prepare_data_list(self): - with self.assertRaises(TypeError): - prepare_data([]) - - def test_prepare_data_none(self): - with self.assertRaises(TypeError): - prepare_data(None) - - def test_encode_data_str(self): - self.assertEqual(encode_data("café"), b"caf\xc3\xa9") - - def test_encode_data_bytes(self): - self.assertEqual(encode_data(b"tea"), b"tea") - - def test_encode_data_bytearray(self): - self.assertEqual(encode_data(bytearray(b"tea")), b"tea") - - def test_encode_data_memoryview(self): - self.assertEqual(encode_data(memoryview(b"tea")), b"tea") - - def test_encode_data_non_contiguous_memoryview(self): - self.assertEqual(encode_data(memoryview(b"tteeaa")[::2]), b"tea") - - def test_encode_data_list(self): - with self.assertRaises(TypeError): - encode_data([]) - - def test_encode_data_none(self): - with self.assertRaises(TypeError): - encode_data(None) - def test_fragmented_control_frame(self): # Fin bit correctly set. self.decode(b"\x88\x00") @@ -204,25 +152,6 @@ def test_fragmented_control_frame(self): with self.assertRaises(ProtocolError): self.decode(b"\x08\x00") - def test_parse_close_and_serialize_close(self): - self.round_trip_close(b"\x03\xe8", 1000, "") - self.round_trip_close(b"\x03\xe8OK", 1000, "OK") - - def test_parse_close_empty(self): - self.assertEqual(parse_close(b""), (1005, "")) - - def test_parse_close_errors(self): - with self.assertRaises(ProtocolError): - parse_close(b"\x03") - with self.assertRaises(ProtocolError): - parse_close(b"\x03\xe7") - with self.assertRaises(UnicodeDecodeError): - parse_close(b"\x03\xe8\xff\xff") - - def test_serialize_close_errors(self): - with self.assertRaises(ProtocolError): - serialize_close(999, "") - def test_extensions(self): class Rot13: @staticmethod diff --git a/tests/test_protocol.py b/tests/test_protocol.py index d32c1f72e..91fb02a50 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -6,7 +6,16 @@ import warnings from websockets.exceptions import ConnectionClosed, InvalidState -from websockets.framing import * +from websockets.frames import ( + OP_BINARY, + OP_CLOSE, + OP_CONT, + OP_PING, + OP_PONG, + OP_TEXT, + serialize_close, +) +from websockets.framing import Frame from websockets.protocol import State, WebSocketCommonProtocol from .utils import MS, AsyncioTestCase From 7b67307ec9f324535cea7e141c6d1a43cb47f4ff Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 14 Oct 2019 21:55:06 +0200 Subject: [PATCH 024/104] Add a sans-I/O compatible HTTP/1.1 implementation. --- src/websockets/http11.py | 295 +++++++++++++++++++++++++++++++++++++++ tests/test_http11.py | 271 +++++++++++++++++++++++++++++++++++ 2 files changed, 566 insertions(+) create mode 100644 src/websockets/http11.py create mode 100644 tests/test_http11.py diff --git a/src/websockets/http11.py b/src/websockets/http11.py new file mode 100644 index 000000000..e1d004881 --- /dev/null +++ b/src/websockets/http11.py @@ -0,0 +1,295 @@ +import re +from typing import Callable, Generator, NamedTuple, Optional + +from .datastructures import Headers +from .exceptions import SecurityError + + +MAX_HEADERS = 256 +MAX_LINE = 4096 + + +def d(value: bytes) -> str: + """ + Decode a bytestring for interpolating into an error message. + + """ + return value.decode(errors="backslashreplace") + + +# See https://tools.ietf.org/html/rfc7230#appendix-B. + +# Regex for validating header names. + +_token_re = re.compile(rb"[-!#$%&\'*+.^_`|~0-9a-zA-Z]+") + +# Regex for validating header values. + +# We don't attempt to support obsolete line folding. + +# Include HTAB (\x09), SP (\x20), VCHAR (\x21-\x7e), obs-text (\x80-\xff). + +# The ABNF is complicated because it attempts to express that optional +# whitespace is ignored. We strip whitespace and don't revalidate that. + +# See also https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 + +_value_re = re.compile(rb"[\x09\x20-\x7e\x80-\xff]*") + + +# Consider converting to dataclasses when dropping support for Python < 3.7. + + +class Request(NamedTuple): + """ + WebSocket handshake request. + + :param path: path and optional query + :param headers: + """ + + path: str + headers: Headers + # body isn't useful is the context of this library + + @classmethod + def parse( + cls, read_line: Callable[[], Generator[None, None, bytes]] + ) -> Generator[None, None, "Request"]: + """ + Parse an HTTP/1.1 GET request and return ``(path, headers)``. + + ``path`` isn't URL-decoded or validated in any way. + + ``path`` and ``headers`` are expected to contain only ASCII characters. + Other characters are represented with surrogate escapes. + + :func:`parse_request` doesn't attempt to read the request body because + WebSocket handshake requests don't have one. If the request contains a + body, it may be read from ``stream`` after this coroutine returns. + + :param read_line: generator-based coroutine that reads a LF-terminated + line or raises an exception if there isn't enough data + :raises EOFError: if the connection is closed without a full HTTP request + :raises SecurityError: if the request exceeds a security limit + :raises ValueError: if the request isn't well formatted + + """ + # https://tools.ietf.org/html/rfc7230#section-3.1.1 + + # Parsing is simple because fixed values are expected for method and + # version and because path isn't checked. Since WebSocket software tends + # to implement HTTP/1.1 strictly, there's little need for lenient parsing. + + try: + request_line = yield from parse_line(read_line) + except EOFError as exc: + raise EOFError("connection closed while reading HTTP request line") from exc + + try: + method, raw_path, version = request_line.split(b" ", 2) + except ValueError: # not enough values to unpack (expected 3, got 1-2) + raise ValueError(f"invalid HTTP request line: {d(request_line)}") from None + + if method != b"GET": + raise ValueError(f"unsupported HTTP method: {d(method)}") + if version != b"HTTP/1.1": + raise ValueError(f"unsupported HTTP version: {d(version)}") + path = raw_path.decode("ascii", "surrogateescape") + + headers = yield from parse_headers(read_line) + + return cls(path, headers) + + def serialize(self) -> bytes: + """ + Serialize an HTTP/1.1 GET request. + + """ + # Since the path and headers only contain ASCII characters, + # we can keep this simple. + request = f"GET {self.path} HTTP/1.1\r\n".encode() + request += self.headers.serialize() + return request + + +# Consider converting to dataclasses when dropping support for Python < 3.7. + + +class Response(NamedTuple): + """ + WebSocket handshake response. + + """ + + status_code: int + reason_phrase: str + headers: Headers + body: Optional[bytes] = None + + @classmethod + def parse( + cls, + read_line: Callable[[], Generator[None, None, bytes]], + read_exact: Callable[[int], Generator[None, None, bytes]], + read_to_eof: Callable[[], Generator[None, None, bytes]], + ) -> Generator[None, None, "Response"]: + """ + Parse an HTTP/1.1 response and return ``(status_code, reason, headers)``. + + ``reason`` and ``headers`` are expected to contain only ASCII characters. + Other characters are represented with surrogate escapes. + + :func:`parse_request` doesn't attempt to read the response body because + WebSocket handshake responses don't have one. If the response contains a + body, it may be read from ``stream`` after this coroutine returns. + + :param read_line: generator-based coroutine that reads a LF-terminated + line or raises an exception if there isn't enough data + :param read_exact: generator-based coroutine that reads the requested + number of bytes or raises an exception if there isn't enough data + :raises EOFError: if the connection is closed without a full HTTP response + :raises SecurityError: if the response exceeds a security limit + :raises LookupError: if the response isn't well formatted + :raises ValueError: if the response isn't well formatted + + """ + # https://tools.ietf.org/html/rfc7230#section-3.1.2 + + # As in parse_request, parsing is simple because a fixed value is expected + # for version, status_code is a 3-digit number, and reason can be ignored. + + try: + status_line = yield from parse_line(read_line) + except EOFError as exc: + raise EOFError("connection closed while reading HTTP status line") from exc + + try: + version, raw_status_code, raw_reason = status_line.split(b" ", 2) + except ValueError: # not enough values to unpack (expected 3, got 1-2) + raise ValueError(f"invalid HTTP status line: {d(status_line)}") from None + + if version != b"HTTP/1.1": + raise ValueError(f"unsupported HTTP version: {d(version)}") + try: + status_code = int(raw_status_code) + except ValueError: # invalid literal for int() with base 10 + raise ValueError( + f"invalid HTTP status code: {d(raw_status_code)}" + ) from None + if not 100 <= status_code < 1000: + raise ValueError(f"unsupported HTTP status code: {d(raw_status_code)}") + if not _value_re.fullmatch(raw_reason): + raise ValueError(f"invalid HTTP reason phrase: {d(raw_reason)}") + reason = raw_reason.decode() + + headers = yield from parse_headers(read_line) + + # https://tools.ietf.org/html/rfc7230#section-3.3.3 + + if "Transfer-Encoding" in headers: + raise NotImplementedError("transfer codings aren't supported") + + # Since websockets only does GET requests (no HEAD, no CONNECT), all + # responses except 1xx, 204, and 304 include a message body. + if 100 <= status_code < 200 or status_code == 204 or status_code == 304: + body = None + else: + content_length: Optional[int] + try: + # MultipleValuesError is sufficiently unlikely that we don't + # attempt to handle it. Instead we document that its parent + # class, LookupError, may be raised. + raw_content_length = headers["Content-Length"] + except KeyError: + content_length = None + else: + content_length = int(raw_content_length) + + if content_length is None: + body = yield from read_to_eof() + else: + body = yield from read_exact(content_length) + + return cls(status_code, reason, headers, body) + + def serialize(self) -> bytes: + """ + Serialize an HTTP/1.1 GET response. + + """ + # Since the status line and headers only contain ASCII characters, + # we can keep this simple. + response = f"HTTP/1.1 {self.status_code} {self.reason_phrase}\r\n".encode() + response += self.headers.serialize() + if self.body is not None: + response += self.body + return response + + +def parse_headers( + read_line: Callable[[], Generator[None, None, bytes]] +) -> Generator[None, None, Headers]: + """ + Parse HTTP headers. + + Non-ASCII characters are represented with surrogate escapes. + + :param read_line: generator-based coroutine that reads a LF-terminated + line or raises an exception if there isn't enough data + + """ + # https://tools.ietf.org/html/rfc7230#section-3.2 + + # We don't attempt to support obsolete line folding. + + headers = Headers() + for _ in range(MAX_HEADERS + 1): + try: + line = yield from parse_line(read_line) + except EOFError as exc: + raise EOFError("connection closed while reading HTTP headers") from exc + if line == b"": + break + + try: + raw_name, raw_value = line.split(b":", 1) + except ValueError: # not enough values to unpack (expected 2, got 1) + raise ValueError(f"invalid HTTP header line: {d(line)}") from None + if not _token_re.fullmatch(raw_name): + raise ValueError(f"invalid HTTP header name: {d(raw_name)}") + raw_value = raw_value.strip(b" \t") + if not _value_re.fullmatch(raw_value): + raise ValueError(f"invalid HTTP header value: {d(raw_value)}") + + name = raw_name.decode("ascii") # guaranteed to be ASCII at this point + value = raw_value.decode("ascii", "surrogateescape") + headers[name] = value + + else: + raise SecurityError("too many HTTP headers") + + return headers + + +def parse_line( + read_line: Callable[[], Generator[None, None, bytes]] +) -> Generator[None, None, bytes]: + """ + Parse a single line. + + CRLF is stripped from the return value. + + :param read_line: generator-based coroutine that reads a LF-terminated + line or raises an exception if there isn't enough data + + """ + # Security: TODO: add a limit here + line = yield from read_line() + # Security: this guarantees header values are small (hard-coded = 4 KiB) + if len(line) > MAX_LINE: + raise SecurityError("line too long") + # Not mandatory but safe - https://tools.ietf.org/html/rfc7230#section-3.5 + if not line.endswith(b"\r\n"): + raise EOFError("line without CRLF") + return line[:-2] diff --git a/tests/test_http11.py b/tests/test_http11.py new file mode 100644 index 000000000..bca874aee --- /dev/null +++ b/tests/test_http11.py @@ -0,0 +1,271 @@ +from websockets.datastructures import Headers +from websockets.exceptions import SecurityError +from websockets.http11 import * +from websockets.http11 import parse_headers +from websockets.streams import StreamReader + +from .utils import GeneratorTestCase + + +class RequestTests(GeneratorTestCase): + def setUp(self): + super().setUp() + self.reader = StreamReader() + + def parse(self): + return Request.parse(self.reader.read_line) + + def test_parse(self): + # Example from the protocol overview in RFC 6455 + self.reader.feed_data( + b"GET /chat HTTP/1.1\r\n" + b"Host: server.example.com\r\n" + b"Upgrade: websocket\r\n" + b"Connection: Upgrade\r\n" + b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + b"Origin: http://example.com\r\n" + b"Sec-WebSocket-Protocol: chat, superchat\r\n" + b"Sec-WebSocket-Version: 13\r\n" + b"\r\n" + ) + request = self.assertGeneratorReturns(self.parse()) + self.assertEqual(request.path, "/chat") + self.assertEqual(request.headers["Upgrade"], "websocket") + + def test_parse_empty(self): + self.reader.feed_eof() + with self.assertRaises(EOFError) as raised: + next(self.parse()) + self.assertEqual( + str(raised.exception), "connection closed while reading HTTP request line" + ) + + def test_parse_invalid_request_line(self): + self.reader.feed_data(b"GET /\r\n\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "invalid HTTP request line: GET /") + + def test_parse_unsupported_method(self): + self.reader.feed_data(b"OPTIONS * HTTP/1.1\r\n\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "unsupported HTTP method: OPTIONS") + + def test_parse_unsupported_version(self): + self.reader.feed_data(b"GET /chat HTTP/1.0\r\n\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "unsupported HTTP version: HTTP/1.0") + + def test_parse_invalid_header(self): + self.reader.feed_data(b"GET /chat HTTP/1.1\r\nOops\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "invalid HTTP header line: Oops") + + def test_serialize(self): + # Example from the protocol overview in RFC 6455 + request = Request( + "/chat", + Headers( + [ + ("Host", "server.example.com"), + ("Upgrade", "websocket"), + ("Connection", "Upgrade"), + ("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ=="), + ("Origin", "http://example.com"), + ("Sec-WebSocket-Protocol", "chat, superchat"), + ("Sec-WebSocket-Version", "13"), + ] + ), + ) + self.assertEqual( + request.serialize(), + b"GET /chat HTTP/1.1\r\n" + b"Host: server.example.com\r\n" + b"Upgrade: websocket\r\n" + b"Connection: Upgrade\r\n" + b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + b"Origin: http://example.com\r\n" + b"Sec-WebSocket-Protocol: chat, superchat\r\n" + b"Sec-WebSocket-Version: 13\r\n" + b"\r\n", + ) + + +class ResponseTests(GeneratorTestCase): + def setUp(self): + super().setUp() + self.reader = StreamReader() + + def parse(self): + return Response.parse( + self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof + ) + + def test_parse(self): + # Example from the protocol overview in RFC 6455 + self.reader.feed_data( + b"HTTP/1.1 101 Switching Protocols\r\n" + b"Upgrade: websocket\r\n" + b"Connection: Upgrade\r\n" + b"Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n" + b"Sec-WebSocket-Protocol: chat\r\n" + b"\r\n" + ) + response = self.assertGeneratorReturns(self.parse()) + self.assertEqual(response.status_code, 101) + self.assertEqual(response.reason_phrase, "Switching Protocols") + self.assertEqual(response.headers["Upgrade"], "websocket") + self.assertIsNone(response.body) + + def test_parse_empty(self): + self.reader.feed_eof() + with self.assertRaises(EOFError) as raised: + next(self.parse()) + self.assertEqual( + str(raised.exception), "connection closed while reading HTTP status line" + ) + + def test_parse_invalid_status_line(self): + self.reader.feed_data(b"Hello!\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "invalid HTTP status line: Hello!") + + def test_parse_unsupported_version(self): + self.reader.feed_data(b"HTTP/1.0 400 Bad Request\r\n\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "unsupported HTTP version: HTTP/1.0") + + def test_parse_invalid_status(self): + self.reader.feed_data(b"HTTP/1.1 OMG WTF\r\n\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "invalid HTTP status code: OMG") + + def test_parse_unsupported_status(self): + self.reader.feed_data(b"HTTP/1.1 007 My name is Bond\r\n\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "unsupported HTTP status code: 007") + + def test_parse_invalid_reason(self): + self.reader.feed_data(b"HTTP/1.1 200 \x7f\r\n\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "invalid HTTP reason phrase: \x7f") + + def test_parse_invalid_header(self): + self.reader.feed_data(b"HTTP/1.1 500 Internal Server Error\r\nOops\r\n") + with self.assertRaises(ValueError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "invalid HTTP header line: Oops") + + def test_parse_body_with_content_length(self): + self.reader.feed_data( + b"HTTP/1.1 200 OK\r\nContent-Length: 13\r\n\r\nHello world!\n" + ) + response = self.assertGeneratorReturns(self.parse()) + self.assertEqual(response.body, b"Hello world!\n") + + def test_parse_body_without_content_length(self): + self.reader.feed_data(b"HTTP/1.1 200 OK\r\n\r\nHello world!\n") + gen = self.parse() + self.assertGeneratorRunning(gen) + self.reader.feed_eof() + response = self.assertGeneratorReturns(gen) + self.assertEqual(response.body, b"Hello world!\n") + + def test_parse_body_with_transfer_encoding(self): + self.reader.feed_data(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") + with self.assertRaises(NotImplementedError) as raised: + next(self.parse()) + self.assertEqual(str(raised.exception), "transfer codings aren't supported") + + def test_parse_body_no_content(self): + self.reader.feed_data(b"HTTP/1.1 204 No Content\r\n\r\n") + response = self.assertGeneratorReturns(self.parse()) + self.assertIsNone(response.body) + + def test_parse_body_not_modified(self): + self.reader.feed_data(b"HTTP/1.1 304 Not Modified\r\n\r\n") + response = self.assertGeneratorReturns(self.parse()) + self.assertIsNone(response.body) + + def test_serialize(self): + # Example from the protocol overview in RFC 6455 + response = Response( + 101, + "Switching Protocols", + Headers( + [ + ("Upgrade", "websocket"), + ("Connection", "Upgrade"), + ("Sec-WebSocket-Accept", "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="), + ("Sec-WebSocket-Protocol", "chat"), + ] + ), + ) + self.assertEqual( + response.serialize(), + b"HTTP/1.1 101 Switching Protocols\r\n" + b"Upgrade: websocket\r\n" + b"Connection: Upgrade\r\n" + b"Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n" + b"Sec-WebSocket-Protocol: chat\r\n" + b"\r\n", + ) + + def test_serialize_with_body(self): + response = Response( + 200, + "OK", + Headers([("Content-Length", "13"), ("Content-Type", "text/plain")]), + b"Hello world!\n", + ) + self.assertEqual( + response.serialize(), + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 13\r\n" + b"Content-Type: text/plain\r\n" + b"\r\n" + b"Hello world!\n", + ) + + +class HeadersTests(GeneratorTestCase): + def setUp(self): + super().setUp() + self.reader = StreamReader() + + def parse_headers(self): + return parse_headers(self.reader.read_line) + + def test_parse_invalid_name(self): + self.reader.feed_data(b"foo bar: baz qux\r\n\r\n") + with self.assertRaises(ValueError): + next(self.parse_headers()) + + def test_parse_invalid_value(self): + self.reader.feed_data(b"foo: \x00\x00\x0f\r\n\r\n") + with self.assertRaises(ValueError): + next(self.parse_headers()) + + def test_parse_too_long_value(self): + self.reader.feed_data(b"foo: bar\r\n" * 257 + b"\r\n") + with self.assertRaises(SecurityError): + next(self.parse_headers()) + + def test_parse_too_long_line(self): + # Header line contains 5 + 4090 + 2 = 4097 bytes. + self.reader.feed_data(b"foo: " + b"a" * 4090 + b"\r\n\r\n") + with self.assertRaises(SecurityError): + next(self.parse_headers()) + + def test_parse_invalid_line_ending(self): + self.reader.feed_data(b"foo: bar\n\n") + with self.assertRaises(EOFError): + next(self.parse_headers()) From e4bc504a880110b0d3cd1dbc8e55b69a5f44ee7c Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 6 Oct 2019 20:00:10 +0200 Subject: [PATCH 025/104] Salvage accept() from the legacy handshake module. --- src/websockets/handshake.py | 3 --- src/websockets/handshake_legacy.py | 8 +------- src/websockets/utils.py | 18 +++++++++++++++++- tests/test_handshake_legacy.py | 10 ++-------- tests/test_utils.py | 16 +++++++++++++--- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/websockets/handshake.py b/src/websockets/handshake.py index f27bd1b84..3ff6c005d 100644 --- a/src/websockets/handshake.py +++ b/src/websockets/handshake.py @@ -6,9 +6,6 @@ __all__ = ["build_request", "check_request", "build_response", "check_response"] -GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" - - # Backwards compatibility with previously documented public APIs diff --git a/src/websockets/handshake_legacy.py b/src/websockets/handshake_legacy.py index 9683e8556..1f6c58e1b 100644 --- a/src/websockets/handshake_legacy.py +++ b/src/websockets/handshake_legacy.py @@ -27,15 +27,14 @@ import base64 import binascii -import hashlib import random from typing import List from .datastructures import Headers, MultipleValuesError from .exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade -from .handshake import GUID from .headers import parse_connection, parse_upgrade from .typing import ConnectionOption, UpgradeProtocol +from .utils import accept_key as accept __all__ = ["build_request", "check_request", "build_response", "check_response"] @@ -180,8 +179,3 @@ def check_response(headers: Headers, key: str) -> None: if s_w_accept != accept(key): raise InvalidHeaderValue("Sec-WebSocket-Accept", s_w_accept) - - -def accept(key: str) -> str: - sha1 = hashlib.sha1((key + GUID).encode()).digest() - return base64.b64encode(sha1).decode() diff --git a/src/websockets/utils.py b/src/websockets/utils.py index 40ac8559f..f9d0ca763 100644 --- a/src/websockets/utils.py +++ b/src/websockets/utils.py @@ -1,7 +1,23 @@ +import base64 +import hashlib import itertools -__all__ = ["apply_mask"] +__all__ = ["accept_key", "apply_mask"] + + +GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +def accept_key(key: str) -> str: + """ + Compute the value of the Sec-WebSocket-Accept header. + + :param key: value of the Sec-WebSocket-Key header + + """ + sha1 = hashlib.sha1((key + GUID).encode()).digest() + return base64.b64encode(sha1).decode() def apply_mask(data: bytes, mask: bytes) -> bytes: diff --git a/tests/test_handshake_legacy.py b/tests/test_handshake_legacy.py index 361410d3f..c34b94e41 100644 --- a/tests/test_handshake_legacy.py +++ b/tests/test_handshake_legacy.py @@ -9,16 +9,10 @@ InvalidUpgrade, ) from websockets.handshake_legacy import * -from websockets.handshake_legacy import accept # private API +from websockets.utils import accept_key class HandshakeTests(unittest.TestCase): - def test_accept(self): - # Test vector from RFC 6455 - key = "dGhlIHNhbXBsZSBub25jZQ==" - acc = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" - self.assertEqual(accept(key), acc) - def test_round_trip(self): request_headers = Headers() request_key = build_request(request_headers) @@ -178,7 +172,7 @@ def test_response_invalid_accept(self): with self.assertInvalidResponseHeaders(InvalidHeaderValue) as headers: del headers["Sec-WebSocket-Accept"] other_key = "1Eq4UDEFQYg3YspNgqxv5g==" - headers["Sec-WebSocket-Accept"] = accept(other_key) + headers["Sec-WebSocket-Accept"] = accept_key(other_key) def test_response_missing_accept(self): with self.assertInvalidResponseHeaders(InvalidHeader) as headers: diff --git a/tests/test_utils.py b/tests/test_utils.py index e5570f098..7d5417d79 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,10 +1,20 @@ import itertools import unittest -from websockets.utils import apply_mask as py_apply_mask +from websockets.utils import accept_key, apply_mask as py_apply_mask -class UtilsTests(unittest.TestCase): +# Test vector from RFC 6455 +KEY = "dGhlIHNhbXBsZSBub25jZQ==" +ACCEPT = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" + + +class AcceptKeyTests(unittest.TestCase): + def test_accept_key(self): + self.assertEqual(accept_key(KEY), ACCEPT) + + +class ApplyMaskTests(unittest.TestCase): @staticmethod def apply_mask(*args, **kwargs): return py_apply_mask(*args, **kwargs) @@ -73,7 +83,7 @@ def test_apply_mask_check_mask_length(self): pass else: - class SpeedupsTests(UtilsTests): + class SpeedupsTests(ApplyMaskTests): @staticmethod def apply_mask(*args, **kwargs): return c_apply_mask(*args, **kwargs) From cf5af352200e6800f0152c8399af067a45053d76 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 25 Jan 2020 21:32:21 +0100 Subject: [PATCH 026/104] Extract generate_key() from the legacy handshake module. --- src/websockets/handshake_legacy.py | 6 ++---- src/websockets/utils.py | 10 ++++++++++ tests/test_utils.py | 9 +++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/websockets/handshake_legacy.py b/src/websockets/handshake_legacy.py index 1f6c58e1b..7e6acc77d 100644 --- a/src/websockets/handshake_legacy.py +++ b/src/websockets/handshake_legacy.py @@ -27,14 +27,13 @@ import base64 import binascii -import random from typing import List from .datastructures import Headers, MultipleValuesError from .exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade from .headers import parse_connection, parse_upgrade from .typing import ConnectionOption, UpgradeProtocol -from .utils import accept_key as accept +from .utils import accept_key as accept, generate_key __all__ = ["build_request", "check_request", "build_response", "check_response"] @@ -50,8 +49,7 @@ def build_request(headers: Headers) -> str: :returns: ``key`` which must be passed to :func:`check_response` """ - raw_key = bytes(random.getrandbits(8) for _ in range(16)) - key = base64.b64encode(raw_key).decode() + key = generate_key() headers["Upgrade"] = "websocket" headers["Connection"] = "Upgrade" headers["Sec-WebSocket-Key"] = key diff --git a/src/websockets/utils.py b/src/websockets/utils.py index f9d0ca763..a2fe8cc7f 100644 --- a/src/websockets/utils.py +++ b/src/websockets/utils.py @@ -1,6 +1,7 @@ import base64 import hashlib import itertools +import random __all__ = ["accept_key", "apply_mask"] @@ -9,6 +10,15 @@ GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" +def generate_key() -> str: + """ + Generate a random key for the Sec-WebSocket-Key header. + + """ + key = bytes(random.getrandbits(8) for _ in range(16)) + return base64.b64encode(key).decode() + + def accept_key(key: str) -> str: """ Compute the value of the Sec-WebSocket-Accept header. diff --git a/tests/test_utils.py b/tests/test_utils.py index 7d5417d79..b490c2409 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,8 @@ +import base64 import itertools import unittest -from websockets.utils import accept_key, apply_mask as py_apply_mask +from websockets.utils import accept_key, apply_mask as py_apply_mask, generate_key # Test vector from RFC 6455 @@ -9,7 +10,11 @@ ACCEPT = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" -class AcceptKeyTests(unittest.TestCase): +class UtilsTests(unittest.TestCase): + def test_generate_key(self): + key = generate_key() + self.assertEqual(len(base64.b64decode(key.encode())), 16) + def test_accept_key(self): self.assertEqual(accept_key(KEY), ACCEPT) From 1af2296159b0e5165bbcf4b636ed7a06520928ab Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 14 Jun 2020 10:27:30 +0200 Subject: [PATCH 027/104] Take advantage of the secrets module. Per RFC 6455, "the masking key MUST be derived from a strong source of entropy." There is no such requirement Sec-WebSocket-Key but it seems better anyway. --- src/websockets/frames.py | 12 ++++++------ src/websockets/utils.py | 4 ++-- tests/test_frames.py | 5 ++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/websockets/frames.py b/src/websockets/frames.py index 5ed8e483f..56dcf6171 100644 --- a/src/websockets/frames.py +++ b/src/websockets/frames.py @@ -4,7 +4,7 @@ """ import io -import random +import secrets import struct from typing import Callable, Generator, NamedTuple, Optional, Sequence, Tuple @@ -120,12 +120,12 @@ def parse( f"payload length exceeds size limit ({length} > {max_size} bytes)" ) if mask: - mask_bits = yield from read_exact(4) + mask_bytes = yield from read_exact(4) # Read the data. data = yield from read_exact(length) if mask: - data = apply_mask(data, mask_bits) + data = apply_mask(data, mask_bytes) frame = cls(fin, opcode, data, rsv1, rsv2, rsv3) @@ -186,12 +186,12 @@ def serialize( output.write(struct.pack("!BBQ", head1, head2 | 127, length)) if mask: - mask_bits = struct.pack("!I", random.getrandbits(32)) - output.write(mask_bits) + mask_bytes = secrets.token_bytes(4) + output.write(mask_bytes) # Prepare the data. if mask: - data = apply_mask(self.data, mask_bits) + data = apply_mask(self.data, mask_bytes) else: data = self.data output.write(data) diff --git a/src/websockets/utils.py b/src/websockets/utils.py index a2fe8cc7f..59210e438 100644 --- a/src/websockets/utils.py +++ b/src/websockets/utils.py @@ -1,7 +1,7 @@ import base64 import hashlib import itertools -import random +import secrets __all__ = ["accept_key", "apply_mask"] @@ -15,7 +15,7 @@ def generate_key() -> str: Generate a random key for the Sec-WebSocket-Key header. """ - key = bytes(random.getrandbits(8) for _ in range(16)) + key = secrets.token_bytes(16) return base64.b64encode(key).decode() diff --git a/tests/test_frames.py b/tests/test_frames.py index 39d4055a8..37a73b2df 100644 --- a/tests/test_frames.py +++ b/tests/test_frames.py @@ -1,5 +1,4 @@ import codecs -import struct import unittest import unittest.mock @@ -26,8 +25,8 @@ def round_trip(self, data, frame, mask=False, extensions=None): # Make masking deterministic by reusing the same "random" mask. # This has an effect only when mask is True. - randbits = struct.unpack("!I", data[2:6])[0] if mask else 0 - with unittest.mock.patch("random.getrandbits", return_value=randbits): + mask_bytes = data[2:6] if mask else b"" + with unittest.mock.patch("secrets.token_bytes", return_value=mask_bytes): serialized = parsed.serialize(mask=mask, extensions=extensions) self.assertEqual(serialized, data) From 6bce2489660daf09f1e6bdf121cabdea83128e4e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 Feb 2020 09:40:33 +0100 Subject: [PATCH 028/104] Move asyncio client and server out of the way. --- src/websockets/asyncio_client.py | 588 ++++++++++ src/websockets/asyncio_server.py | 1004 ++++++++++++++++ src/websockets/auth.py | 2 +- src/websockets/client.py | 592 +--------- src/websockets/server.py | 1009 +---------------- ...erver.py => test_asyncio_client_server.py} | 28 +- tests/test_auth.py | 2 +- 7 files changed, 1623 insertions(+), 1602 deletions(-) create mode 100644 src/websockets/asyncio_client.py create mode 100644 src/websockets/asyncio_server.py rename tests/{test_client_server.py => test_asyncio_client_server.py} (98%) diff --git a/src/websockets/asyncio_client.py b/src/websockets/asyncio_client.py new file mode 100644 index 000000000..f95dae060 --- /dev/null +++ b/src/websockets/asyncio_client.py @@ -0,0 +1,588 @@ +""" +:mod:`websockets.client` defines the WebSocket client APIs. + +""" + +import asyncio +import collections.abc +import functools +import logging +import warnings +from types import TracebackType +from typing import Any, Generator, List, Optional, Sequence, Tuple, Type, cast + +from .datastructures import Headers, HeadersLike +from .exceptions import ( + InvalidHandshake, + InvalidHeader, + InvalidMessage, + InvalidStatusCode, + NegotiationError, + RedirectHandshake, + SecurityError, +) +from .extensions.base import ClientExtensionFactory, Extension +from .extensions.permessage_deflate import ClientPerMessageDeflateFactory +from .handshake_legacy import build_request, check_response +from .headers import ( + build_authorization_basic, + build_extension, + build_subprotocol, + parse_extension, + parse_subprotocol, +) +from .http import USER_AGENT +from .http_legacy import read_response +from .protocol import WebSocketCommonProtocol +from .typing import ExtensionHeader, Origin, Subprotocol +from .uri import WebSocketURI, parse_uri + + +__all__ = ["connect", "unix_connect", "WebSocketClientProtocol"] + +logger = logging.getLogger(__name__) + + +class WebSocketClientProtocol(WebSocketCommonProtocol): + """ + :class:`~asyncio.Protocol` subclass implementing a WebSocket client. + + This class inherits most of its methods from + :class:`~websockets.protocol.WebSocketCommonProtocol`. + + """ + + is_client = True + side = "client" + + def __init__( + self, + *, + origin: Optional[Origin] = None, + extensions: Optional[Sequence[ClientExtensionFactory]] = None, + subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLike] = None, + **kwargs: Any, + ) -> None: + self.origin = origin + self.available_extensions = extensions + self.available_subprotocols = subprotocols + self.extra_headers = extra_headers + super().__init__(**kwargs) + + def write_http_request(self, path: str, headers: Headers) -> None: + """ + Write request line and headers to the HTTP request. + + """ + self.path = path + self.request_headers = headers + + logger.debug("%s > GET %s HTTP/1.1", self.side, path) + logger.debug("%s > %r", self.side, headers) + + # Since the path and headers only contain ASCII characters, + # we can keep this simple. + request = f"GET {path} HTTP/1.1\r\n" + request += str(headers) + + self.transport.write(request.encode()) + + async def read_http_response(self) -> Tuple[int, Headers]: + """ + Read status line and headers from the HTTP response. + + If the response contains a body, it may be read from ``self.reader`` + after this coroutine returns. + + :raises ~websockets.exceptions.InvalidMessage: if the HTTP message is + malformed or isn't an HTTP/1.1 GET response + + """ + try: + status_code, reason, headers = await read_response(self.reader) + except asyncio.CancelledError: # pragma: no cover + raise + except Exception as exc: + raise InvalidMessage("did not receive a valid HTTP response") from exc + + logger.debug("%s < HTTP/1.1 %d %s", self.side, status_code, reason) + logger.debug("%s < %r", self.side, headers) + + self.response_headers = headers + + return status_code, self.response_headers + + @staticmethod + def process_extensions( + headers: Headers, + available_extensions: Optional[Sequence[ClientExtensionFactory]], + ) -> List[Extension]: + """ + Handle the Sec-WebSocket-Extensions HTTP response header. + + Check that each extension is supported, as well as its parameters. + + Return the list of accepted extensions. + + Raise :exc:`~websockets.exceptions.InvalidHandshake` to abort the + connection. + + :rfc:`6455` leaves the rules up to the specification of each + :extension. + + To provide this level of flexibility, for each extension accepted by + the server, we check for a match with each extension available in the + client configuration. If no match is found, an exception is raised. + + If several variants of the same extension are accepted by the server, + it may be configured several times, which won't make sense in general. + Extensions must implement their own requirements. For this purpose, + the list of previously accepted extensions is provided. + + Other requirements, for example related to mandatory extensions or the + order of extensions, may be implemented by overriding this method. + + """ + accepted_extensions: List[Extension] = [] + + header_values = headers.get_all("Sec-WebSocket-Extensions") + + if header_values: + + if available_extensions is None: + raise InvalidHandshake("no extensions supported") + + parsed_header_values: List[ExtensionHeader] = sum( + [parse_extension(header_value) for header_value in header_values], [] + ) + + for name, response_params in parsed_header_values: + + for extension_factory in available_extensions: + + # Skip non-matching extensions based on their name. + if extension_factory.name != name: + continue + + # Skip non-matching extensions based on their params. + try: + extension = extension_factory.process_response_params( + response_params, accepted_extensions + ) + except NegotiationError: + continue + + # Add matching extension to the final list. + accepted_extensions.append(extension) + + # Break out of the loop once we have a match. + break + + # If we didn't break from the loop, no extension in our list + # matched what the server sent. Fail the connection. + else: + raise NegotiationError( + f"Unsupported extension: " + f"name = {name}, params = {response_params}" + ) + + return accepted_extensions + + @staticmethod + def process_subprotocol( + headers: Headers, available_subprotocols: Optional[Sequence[Subprotocol]] + ) -> Optional[Subprotocol]: + """ + Handle the Sec-WebSocket-Protocol HTTP response header. + + Check that it contains exactly one supported subprotocol. + + Return the selected subprotocol. + + """ + subprotocol: Optional[Subprotocol] = None + + header_values = headers.get_all("Sec-WebSocket-Protocol") + + if header_values: + + if available_subprotocols is None: + raise InvalidHandshake("no subprotocols supported") + + parsed_header_values: Sequence[Subprotocol] = sum( + [parse_subprotocol(header_value) for header_value in header_values], [] + ) + + if len(parsed_header_values) > 1: + subprotocols = ", ".join(parsed_header_values) + raise InvalidHandshake(f"multiple subprotocols: {subprotocols}") + + subprotocol = parsed_header_values[0] + + if subprotocol not in available_subprotocols: + raise NegotiationError(f"unsupported subprotocol: {subprotocol}") + + return subprotocol + + async def handshake( + self, + wsuri: WebSocketURI, + origin: Optional[Origin] = None, + available_extensions: Optional[Sequence[ClientExtensionFactory]] = None, + available_subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLike] = None, + ) -> None: + """ + Perform the client side of the opening handshake. + + :param origin: sets the Origin HTTP header + :param available_extensions: list of supported extensions in the order + in which they should be used + :param available_subprotocols: list of supported subprotocols in order + of decreasing preference + :param extra_headers: sets additional HTTP request headers; it must be + a :class:`~websockets.http.Headers` instance, a + :class:`~collections.abc.Mapping`, or an iterable of ``(name, + value)`` pairs + :raises ~websockets.exceptions.InvalidHandshake: if the handshake + fails + + """ + request_headers = Headers() + + if wsuri.port == (443 if wsuri.secure else 80): # pragma: no cover + request_headers["Host"] = wsuri.host + else: + request_headers["Host"] = f"{wsuri.host}:{wsuri.port}" + + if wsuri.user_info: + request_headers["Authorization"] = build_authorization_basic( + *wsuri.user_info + ) + + if origin is not None: + request_headers["Origin"] = origin + + key = build_request(request_headers) + + if available_extensions is not None: + extensions_header = build_extension( + [ + (extension_factory.name, extension_factory.get_request_params()) + for extension_factory in available_extensions + ] + ) + request_headers["Sec-WebSocket-Extensions"] = extensions_header + + if available_subprotocols is not None: + protocol_header = build_subprotocol(available_subprotocols) + request_headers["Sec-WebSocket-Protocol"] = protocol_header + + if extra_headers is not None: + if isinstance(extra_headers, Headers): + extra_headers = extra_headers.raw_items() + elif isinstance(extra_headers, collections.abc.Mapping): + extra_headers = extra_headers.items() + for name, value in extra_headers: + request_headers[name] = value + + request_headers.setdefault("User-Agent", USER_AGENT) + + self.write_http_request(wsuri.resource_name, request_headers) + + status_code, response_headers = await self.read_http_response() + if status_code in (301, 302, 303, 307, 308): + if "Location" not in response_headers: + raise InvalidHeader("Location") + raise RedirectHandshake(response_headers["Location"]) + elif status_code != 101: + raise InvalidStatusCode(status_code) + + check_response(response_headers, key) + + self.extensions = self.process_extensions( + response_headers, available_extensions + ) + + self.subprotocol = self.process_subprotocol( + response_headers, available_subprotocols + ) + + self.connection_open() + + +class Connect: + """ + Connect to the WebSocket server at the given ``uri``. + + Awaiting :func:`connect` yields a :class:`WebSocketClientProtocol` which + can then be used to send and receive messages. + + :func:`connect` can also be used as a asynchronous context manager. In + that case, the connection is closed when exiting the context. + + :func:`connect` is a wrapper around the event loop's + :meth:`~asyncio.loop.create_connection` method. Unknown keyword arguments + are passed to :meth:`~asyncio.loop.create_connection`. + + For example, you can set the ``ssl`` keyword argument to a + :class:`~ssl.SSLContext` to enforce some TLS settings. When connecting to + a ``wss://`` URI, if this argument isn't provided explicitly, + :func:`ssl.create_default_context` is called to create a context. + + You can connect to a different host and port from those found in ``uri`` + by setting ``host`` and ``port`` keyword arguments. This only changes the + destination of the TCP connection. The host name from ``uri`` is still + used in the TLS handshake for secure connections and in the ``Host`` HTTP + header. + + The ``create_protocol`` parameter allows customizing the + :class:`~asyncio.Protocol` that manages the connection. It should be a + callable or class accepting the same arguments as + :class:`WebSocketClientProtocol` and returning an instance of + :class:`WebSocketClientProtocol` or a subclass. It defaults to + :class:`WebSocketClientProtocol`. + + The behavior of ``ping_interval``, ``ping_timeout``, ``close_timeout``, + ``max_size``, ``max_queue``, ``read_limit``, and ``write_limit`` is + described in :class:`~websockets.protocol.WebSocketCommonProtocol`. + + :func:`connect` also accepts the following optional arguments: + + * ``compression`` is a shortcut to configure compression extensions; + by default it enables the "permessage-deflate" extension; set it to + ``None`` to disable compression + * ``origin`` sets the Origin HTTP header + * ``extensions`` is a list of supported extensions in order of + decreasing preference + * ``subprotocols`` is a list of supported subprotocols in order of + decreasing preference + * ``extra_headers`` sets additional HTTP request headers; it can be a + :class:`~websockets.http.Headers` instance, a + :class:`~collections.abc.Mapping`, or an iterable of ``(name, value)`` + pairs + + :raises ~websockets.uri.InvalidURI: if ``uri`` is invalid + :raises ~websockets.handshake.InvalidHandshake: if the opening handshake + fails + + """ + + MAX_REDIRECTS_ALLOWED = 10 + + def __init__( + self, + uri: str, + *, + path: Optional[str] = None, + create_protocol: Optional[Type[WebSocketClientProtocol]] = None, + ping_interval: Optional[float] = 20, + ping_timeout: Optional[float] = 20, + close_timeout: Optional[float] = None, + max_size: Optional[int] = 2 ** 20, + max_queue: Optional[int] = 2 ** 5, + read_limit: int = 2 ** 16, + write_limit: int = 2 ** 16, + loop: Optional[asyncio.AbstractEventLoop] = None, + legacy_recv: bool = False, + klass: Optional[Type[WebSocketClientProtocol]] = None, + timeout: Optional[float] = None, + compression: Optional[str] = "deflate", + origin: Optional[Origin] = None, + extensions: Optional[Sequence[ClientExtensionFactory]] = None, + subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLike] = None, + **kwargs: Any, + ) -> None: + # Backwards compatibility: close_timeout used to be called timeout. + if timeout is None: + timeout = 10 + else: + warnings.warn("rename timeout to close_timeout", DeprecationWarning) + # If both are specified, timeout is ignored. + if close_timeout is None: + close_timeout = timeout + + # Backwards compatibility: create_protocol used to be called klass. + if klass is None: + klass = WebSocketClientProtocol + else: + warnings.warn("rename klass to create_protocol", DeprecationWarning) + # If both are specified, klass is ignored. + if create_protocol is None: + create_protocol = klass + + if loop is None: + loop = asyncio.get_event_loop() + + wsuri = parse_uri(uri) + if wsuri.secure: + kwargs.setdefault("ssl", True) + elif kwargs.get("ssl") is not None: + raise ValueError( + "connect() received a ssl argument for a ws:// URI, " + "use a wss:// URI to enable TLS" + ) + + if compression == "deflate": + if extensions is None: + extensions = [] + if not any( + extension_factory.name == ClientPerMessageDeflateFactory.name + for extension_factory in extensions + ): + extensions = list(extensions) + [ + ClientPerMessageDeflateFactory(client_max_window_bits=True) + ] + elif compression is not None: + raise ValueError(f"unsupported compression: {compression}") + + factory = functools.partial( + create_protocol, + ping_interval=ping_interval, + ping_timeout=ping_timeout, + close_timeout=close_timeout, + max_size=max_size, + max_queue=max_queue, + read_limit=read_limit, + write_limit=write_limit, + loop=loop, + host=wsuri.host, + port=wsuri.port, + secure=wsuri.secure, + legacy_recv=legacy_recv, + origin=origin, + extensions=extensions, + subprotocols=subprotocols, + extra_headers=extra_headers, + ) + + if path is None: + host: Optional[str] + port: Optional[int] + if kwargs.get("sock") is None: + host, port = wsuri.host, wsuri.port + else: + # If sock is given, host and port shouldn't be specified. + host, port = None, None + # If host and port are given, override values from the URI. + host = kwargs.pop("host", host) + port = kwargs.pop("port", port) + create_connection = functools.partial( + loop.create_connection, factory, host, port, **kwargs + ) + else: + create_connection = functools.partial( + loop.create_unix_connection, factory, path, **kwargs + ) + + # This is a coroutine function. + self._create_connection = create_connection + self._wsuri = wsuri + + def handle_redirect(self, uri: str) -> None: + # Update the state of this instance to connect to a new URI. + old_wsuri = self._wsuri + new_wsuri = parse_uri(uri) + + # Forbid TLS downgrade. + if old_wsuri.secure and not new_wsuri.secure: + raise SecurityError("redirect from WSS to WS") + + same_origin = ( + old_wsuri.host == new_wsuri.host and old_wsuri.port == new_wsuri.port + ) + + # Rewrite the host and port arguments for cross-origin redirects. + # This preserves connection overrides with the host and port + # arguments if the redirect points to the same host and port. + if not same_origin: + # Replace the host and port argument passed to the protocol factory. + factory = self._create_connection.args[0] + factory = functools.partial( + factory.func, + *factory.args, + **dict(factory.keywords, host=new_wsuri.host, port=new_wsuri.port), + ) + # Replace the host and port argument passed to create_connection. + self._create_connection = functools.partial( + self._create_connection.func, + *(factory, new_wsuri.host, new_wsuri.port), + **self._create_connection.keywords, + ) + + # Set the new WebSocket URI. This suffices for same-origin redirects. + self._wsuri = new_wsuri + + # async with connect(...) + + async def __aenter__(self) -> WebSocketClientProtocol: + return await self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + await self.ws_client.close() + + # await connect(...) + + def __await__(self) -> Generator[Any, None, WebSocketClientProtocol]: + # Create a suitable iterator by calling __await__ on a coroutine. + return self.__await_impl__().__await__() + + async def __await_impl__(self) -> WebSocketClientProtocol: + for redirects in range(self.MAX_REDIRECTS_ALLOWED): + transport, protocol = await self._create_connection() + # https://github.com/python/typeshed/pull/2756 + transport = cast(asyncio.Transport, transport) + protocol = cast(WebSocketClientProtocol, protocol) + + try: + try: + await protocol.handshake( + self._wsuri, + origin=protocol.origin, + available_extensions=protocol.available_extensions, + available_subprotocols=protocol.available_subprotocols, + extra_headers=protocol.extra_headers, + ) + except Exception: + protocol.fail_connection() + await protocol.wait_closed() + raise + else: + self.ws_client = protocol + return protocol + except RedirectHandshake as exc: + self.handle_redirect(exc.uri) + else: + raise SecurityError("too many redirects") + + # yield from connect(...) + + __iter__ = __await__ + + +connect = Connect + + +def unix_connect(path: str, uri: str = "ws://localhost/", **kwargs: Any) -> Connect: + """ + Similar to :func:`connect`, but for connecting to a Unix socket. + + This function calls the event loop's + :meth:`~asyncio.loop.create_unix_connection` method. + + It is only available on Unix. + + It's mainly useful for debugging servers listening on Unix sockets. + + :param path: file system path to the Unix socket + :param uri: WebSocket URI + + """ + return connect(uri=uri, path=path, **kwargs) diff --git a/src/websockets/asyncio_server.py b/src/websockets/asyncio_server.py new file mode 100644 index 000000000..1eeddf0eb --- /dev/null +++ b/src/websockets/asyncio_server.py @@ -0,0 +1,1004 @@ +""" +:mod:`websockets.server` defines the WebSocket server APIs. + +""" + +import asyncio +import collections.abc +import email.utils +import functools +import http +import logging +import socket +import sys +import warnings +from types import TracebackType +from typing import ( + Any, + Awaitable, + Callable, + Generator, + List, + Optional, + Sequence, + Set, + Tuple, + Type, + Union, + cast, +) + +from .datastructures import Headers, HeadersLike, MultipleValuesError +from .exceptions import ( + AbortHandshake, + InvalidHandshake, + InvalidHeader, + InvalidMessage, + InvalidOrigin, + InvalidUpgrade, + NegotiationError, +) +from .extensions.base import Extension, ServerExtensionFactory +from .extensions.permessage_deflate import ServerPerMessageDeflateFactory +from .handshake_legacy import build_response, check_request +from .headers import build_extension, parse_extension, parse_subprotocol +from .http import USER_AGENT +from .http_legacy import read_request +from .protocol import WebSocketCommonProtocol +from .typing import ExtensionHeader, Origin, Subprotocol + + +__all__ = ["serve", "unix_serve", "WebSocketServerProtocol", "WebSocketServer"] + +logger = logging.getLogger(__name__) + + +HeadersLikeOrCallable = Union[HeadersLike, Callable[[str, Headers], HeadersLike]] + +HTTPResponse = Tuple[http.HTTPStatus, HeadersLike, bytes] + + +class WebSocketServerProtocol(WebSocketCommonProtocol): + """ + :class:`~asyncio.Protocol` subclass implementing a WebSocket server. + + This class inherits most of its methods from + :class:`~websockets.protocol.WebSocketCommonProtocol`. + + For the sake of simplicity, it doesn't rely on a full HTTP implementation. + Its support for HTTP responses is very limited. + + """ + + is_client = False + side = "server" + + def __init__( + self, + ws_handler: Callable[["WebSocketServerProtocol", str], Awaitable[Any]], + ws_server: "WebSocketServer", + *, + origins: Optional[Sequence[Optional[Origin]]] = None, + extensions: Optional[Sequence[ServerExtensionFactory]] = None, + subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLikeOrCallable] = None, + process_request: Optional[ + Callable[[str, Headers], Awaitable[Optional[HTTPResponse]]] + ] = None, + select_subprotocol: Optional[ + Callable[[Sequence[Subprotocol], Sequence[Subprotocol]], Subprotocol] + ] = None, + **kwargs: Any, + ) -> None: + # For backwards compatibility with 6.0 or earlier. + if origins is not None and "" in origins: + warnings.warn("use None instead of '' in origins", DeprecationWarning) + origins = [None if origin == "" else origin for origin in origins] + self.ws_handler = ws_handler + self.ws_server = ws_server + self.origins = origins + self.available_extensions = extensions + self.available_subprotocols = subprotocols + self.extra_headers = extra_headers + self._process_request = process_request + self._select_subprotocol = select_subprotocol + super().__init__(**kwargs) + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + """ + Register connection and initialize a task to handle it. + + """ + super().connection_made(transport) + # Register the connection with the server before creating the handler + # task. Registering at the beginning of the handler coroutine would + # create a race condition between the creation of the task, which + # schedules its execution, and the moment the handler starts running. + self.ws_server.register(self) + self.handler_task = self.loop.create_task(self.handler()) + + async def handler(self) -> None: + """ + Handle the lifecycle of a WebSocket connection. + + Since this method doesn't have a caller able to handle exceptions, it + attemps to log relevant ones and guarantees that the TCP connection is + closed before exiting. + + """ + try: + + try: + path = await self.handshake( + origins=self.origins, + available_extensions=self.available_extensions, + available_subprotocols=self.available_subprotocols, + extra_headers=self.extra_headers, + ) + except asyncio.CancelledError: # pragma: no cover + raise + except ConnectionError: + logger.debug("Connection error in opening handshake", exc_info=True) + raise + except Exception as exc: + if isinstance(exc, AbortHandshake): + status, headers, body = exc.status, exc.headers, exc.body + elif isinstance(exc, InvalidOrigin): + logger.debug("Invalid origin", exc_info=True) + status, headers, body = ( + http.HTTPStatus.FORBIDDEN, + Headers(), + f"Failed to open a WebSocket connection: {exc}.\n".encode(), + ) + elif isinstance(exc, InvalidUpgrade): + logger.debug("Invalid upgrade", exc_info=True) + status, headers, body = ( + http.HTTPStatus.UPGRADE_REQUIRED, + Headers([("Upgrade", "websocket")]), + ( + f"Failed to open a WebSocket connection: {exc}.\n" + f"\n" + f"You cannot access a WebSocket server directly " + f"with a browser. You need a WebSocket client.\n" + ).encode(), + ) + elif isinstance(exc, InvalidHandshake): + logger.debug("Invalid handshake", exc_info=True) + status, headers, body = ( + http.HTTPStatus.BAD_REQUEST, + Headers(), + f"Failed to open a WebSocket connection: {exc}.\n".encode(), + ) + else: + logger.warning("Error in opening handshake", exc_info=True) + status, headers, body = ( + http.HTTPStatus.INTERNAL_SERVER_ERROR, + Headers(), + ( + b"Failed to open a WebSocket connection.\n" + b"See server log for more information.\n" + ), + ) + + headers.setdefault("Date", email.utils.formatdate(usegmt=True)) + headers.setdefault("Server", USER_AGENT) + headers.setdefault("Content-Length", str(len(body))) + headers.setdefault("Content-Type", "text/plain") + headers.setdefault("Connection", "close") + + self.write_http_response(status, headers, body) + self.fail_connection() + await self.wait_closed() + return + + try: + await self.ws_handler(self, path) + except Exception: + logger.error("Error in connection handler", exc_info=True) + if not self.closed: + self.fail_connection(1011) + raise + + try: + await self.close() + except ConnectionError: + logger.debug("Connection error in closing handshake", exc_info=True) + raise + except Exception: + logger.warning("Error in closing handshake", exc_info=True) + raise + + except Exception: + # Last-ditch attempt to avoid leaking connections on errors. + try: + self.transport.close() + except Exception: # pragma: no cover + pass + + finally: + # Unregister the connection with the server when the handler task + # terminates. Registration is tied to the lifecycle of the handler + # task because the server waits for tasks attached to registered + # connections before terminating. + self.ws_server.unregister(self) + + async def read_http_request(self) -> Tuple[str, Headers]: + """ + Read request line and headers from the HTTP request. + + If the request contains a body, it may be read from ``self.reader`` + after this coroutine returns. + + :raises ~websockets.exceptions.InvalidMessage: if the HTTP message is + malformed or isn't an HTTP/1.1 GET request + + """ + try: + path, headers = await read_request(self.reader) + except asyncio.CancelledError: # pragma: no cover + raise + except Exception as exc: + raise InvalidMessage("did not receive a valid HTTP request") from exc + + logger.debug("%s < GET %s HTTP/1.1", self.side, path) + logger.debug("%s < %r", self.side, headers) + + self.path = path + self.request_headers = headers + + return path, headers + + def write_http_response( + self, status: http.HTTPStatus, headers: Headers, body: Optional[bytes] = None + ) -> None: + """ + Write status line and headers to the HTTP response. + + This coroutine is also able to write a response body. + + """ + self.response_headers = headers + + logger.debug("%s > HTTP/1.1 %d %s", self.side, status.value, status.phrase) + logger.debug("%s > %r", self.side, headers) + + # Since the status line and headers only contain ASCII characters, + # we can keep this simple. + response = f"HTTP/1.1 {status.value} {status.phrase}\r\n" + response += str(headers) + + self.transport.write(response.encode()) + + if body is not None: + logger.debug("%s > body (%d bytes)", self.side, len(body)) + self.transport.write(body) + + async def process_request( + self, path: str, request_headers: Headers + ) -> Optional[HTTPResponse]: + """ + Intercept the HTTP request and return an HTTP response if appropriate. + + If ``process_request`` returns ``None``, the WebSocket handshake + continues. If it returns 3-uple containing a status code, response + headers and a response body, that HTTP response is sent and the + connection is closed. In that case: + + * The HTTP status must be a :class:`~http.HTTPStatus`. + * HTTP headers must be a :class:`~websockets.http.Headers` instance, a + :class:`~collections.abc.Mapping`, or an iterable of ``(name, + value)`` pairs. + * The HTTP response body must be :class:`bytes`. It may be empty. + + This coroutine may be overridden in a :class:`WebSocketServerProtocol` + subclass, for example: + + * to return a HTTP 200 OK response on a given path; then a load + balancer can use this path for a health check; + * to authenticate the request and return a HTTP 401 Unauthorized or a + HTTP 403 Forbidden when authentication fails. + + Instead of subclassing, it is possible to override this method by + passing a ``process_request`` argument to the :func:`serve` function + or the :class:`WebSocketServerProtocol` constructor. This is + equivalent, except ``process_request`` won't have access to the + protocol instance, so it can't store information for later use. + + ``process_request`` is expected to complete quickly. If it may run for + a long time, then it should await :meth:`wait_closed` and exit if + :meth:`wait_closed` completes, or else it could prevent the server + from shutting down. + + :param path: request path, including optional query string + :param request_headers: request headers + + """ + if self._process_request is not None: + response = self._process_request(path, request_headers) + if isinstance(response, Awaitable): + return await response + else: + # For backwards compatibility with 7.0. + warnings.warn( + "declare process_request as a coroutine", DeprecationWarning + ) + return response # type: ignore + return None + + @staticmethod + def process_origin( + headers: Headers, origins: Optional[Sequence[Optional[Origin]]] = None + ) -> Optional[Origin]: + """ + Handle the Origin HTTP request header. + + :param headers: request headers + :param origins: optional list of acceptable origins + :raises ~websockets.exceptions.InvalidOrigin: if the origin isn't + acceptable + + """ + # "The user agent MUST NOT include more than one Origin header field" + # per https://tools.ietf.org/html/rfc6454#section-7.3. + try: + origin = cast(Origin, headers.get("Origin")) + except MultipleValuesError as exc: + raise InvalidHeader("Origin", "more than one Origin header found") from exc + if origins is not None: + if origin not in origins: + raise InvalidOrigin(origin) + return origin + + @staticmethod + def process_extensions( + headers: Headers, + available_extensions: Optional[Sequence[ServerExtensionFactory]], + ) -> Tuple[Optional[str], List[Extension]]: + """ + Handle the Sec-WebSocket-Extensions HTTP request header. + + Accept or reject each extension proposed in the client request. + Negotiate parameters for accepted extensions. + + Return the Sec-WebSocket-Extensions HTTP response header and the list + of accepted extensions. + + :rfc:`6455` leaves the rules up to the specification of each + :extension. + + To provide this level of flexibility, for each extension proposed by + the client, we check for a match with each extension available in the + server configuration. If no match is found, the extension is ignored. + + If several variants of the same extension are proposed by the client, + it may be accepted several times, which won't make sense in general. + Extensions must implement their own requirements. For this purpose, + the list of previously accepted extensions is provided. + + This process doesn't allow the server to reorder extensions. It can + only select a subset of the extensions proposed by the client. + + Other requirements, for example related to mandatory extensions or the + order of extensions, may be implemented by overriding this method. + + :param headers: request headers + :param extensions: optional list of supported extensions + :raises ~websockets.exceptions.InvalidHandshake: to abort the + handshake with an HTTP 400 error code + + """ + response_header_value: Optional[str] = None + + extension_headers: List[ExtensionHeader] = [] + accepted_extensions: List[Extension] = [] + + header_values = headers.get_all("Sec-WebSocket-Extensions") + + if header_values and available_extensions: + + parsed_header_values: List[ExtensionHeader] = sum( + [parse_extension(header_value) for header_value in header_values], [] + ) + + for name, request_params in parsed_header_values: + + for ext_factory in available_extensions: + + # Skip non-matching extensions based on their name. + if ext_factory.name != name: + continue + + # Skip non-matching extensions based on their params. + try: + response_params, extension = ext_factory.process_request_params( + request_params, accepted_extensions + ) + except NegotiationError: + continue + + # Add matching extension to the final list. + extension_headers.append((name, response_params)) + accepted_extensions.append(extension) + + # Break out of the loop once we have a match. + break + + # If we didn't break from the loop, no extension in our list + # matched what the client sent. The extension is declined. + + # Serialize extension header. + if extension_headers: + response_header_value = build_extension(extension_headers) + + return response_header_value, accepted_extensions + + # Not @staticmethod because it calls self.select_subprotocol() + def process_subprotocol( + self, headers: Headers, available_subprotocols: Optional[Sequence[Subprotocol]] + ) -> Optional[Subprotocol]: + """ + Handle the Sec-WebSocket-Protocol HTTP request header. + + Return Sec-WebSocket-Protocol HTTP response header, which is the same + as the selected subprotocol. + + :param headers: request headers + :param available_subprotocols: optional list of supported subprotocols + :raises ~websockets.exceptions.InvalidHandshake: to abort the + handshake with an HTTP 400 error code + + """ + subprotocol: Optional[Subprotocol] = None + + header_values = headers.get_all("Sec-WebSocket-Protocol") + + if header_values and available_subprotocols: + + parsed_header_values: List[Subprotocol] = sum( + [parse_subprotocol(header_value) for header_value in header_values], [] + ) + + subprotocol = self.select_subprotocol( + parsed_header_values, available_subprotocols + ) + + return subprotocol + + def select_subprotocol( + self, + client_subprotocols: Sequence[Subprotocol], + server_subprotocols: Sequence[Subprotocol], + ) -> Optional[Subprotocol]: + """ + Pick a subprotocol among those offered by the client. + + If several subprotocols are supported by the client and the server, + the default implementation selects the preferred subprotocols by + giving equal value to the priorities of the client and the server. + + If no subprotocol is supported by the client and the server, it + proceeds without a subprotocol. + + This is unlikely to be the most useful implementation in practice, as + many servers providing a subprotocol will require that the client uses + that subprotocol. Such rules can be implemented in a subclass. + + Instead of subclassing, it is possible to override this method by + passing a ``select_subprotocol`` argument to the :func:`serve` + function or the :class:`WebSocketServerProtocol` constructor + + :param client_subprotocols: list of subprotocols offered by the client + :param server_subprotocols: list of subprotocols available on the server + + """ + if self._select_subprotocol is not None: + return self._select_subprotocol(client_subprotocols, server_subprotocols) + + subprotocols = set(client_subprotocols) & set(server_subprotocols) + if not subprotocols: + return None + priority = lambda p: ( + client_subprotocols.index(p) + server_subprotocols.index(p) + ) + return sorted(subprotocols, key=priority)[0] + + async def handshake( + self, + origins: Optional[Sequence[Optional[Origin]]] = None, + available_extensions: Optional[Sequence[ServerExtensionFactory]] = None, + available_subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLikeOrCallable] = None, + ) -> str: + """ + Perform the server side of the opening handshake. + + Return the path of the URI of the request. + + :param origins: list of acceptable values of the Origin HTTP header; + include ``None`` if the lack of an origin is acceptable + :param available_extensions: list of supported extensions in the order + in which they should be used + :param available_subprotocols: list of supported subprotocols in order + of decreasing preference + :param extra_headers: sets additional HTTP response headers when the + handshake succeeds; it can be a :class:`~websockets.http.Headers` + instance, a :class:`~collections.abc.Mapping`, an iterable of + ``(name, value)`` pairs, or a callable taking the request path and + headers in arguments and returning one of the above. + :raises ~websockets.exceptions.InvalidHandshake: if the handshake + fails + + """ + path, request_headers = await self.read_http_request() + + # Hook for customizing request handling, for example checking + # authentication or treating some paths as plain HTTP endpoints. + early_response_awaitable = self.process_request(path, request_headers) + if isinstance(early_response_awaitable, Awaitable): + early_response = await early_response_awaitable + else: + # For backwards compatibility with 7.0. + warnings.warn("declare process_request as a coroutine", DeprecationWarning) + early_response = early_response_awaitable # type: ignore + + # Change the response to a 503 error if the server is shutting down. + if not self.ws_server.is_serving(): + early_response = ( + http.HTTPStatus.SERVICE_UNAVAILABLE, + [], + b"Server is shutting down.\n", + ) + + if early_response is not None: + raise AbortHandshake(*early_response) + + key = check_request(request_headers) + + self.origin = self.process_origin(request_headers, origins) + + extensions_header, self.extensions = self.process_extensions( + request_headers, available_extensions + ) + + protocol_header = self.subprotocol = self.process_subprotocol( + request_headers, available_subprotocols + ) + + response_headers = Headers() + + build_response(response_headers, key) + + if extensions_header is not None: + response_headers["Sec-WebSocket-Extensions"] = extensions_header + + if protocol_header is not None: + response_headers["Sec-WebSocket-Protocol"] = protocol_header + + if callable(extra_headers): + extra_headers = extra_headers(path, self.request_headers) + if extra_headers is not None: + if isinstance(extra_headers, Headers): + extra_headers = extra_headers.raw_items() + elif isinstance(extra_headers, collections.abc.Mapping): + extra_headers = extra_headers.items() + for name, value in extra_headers: + response_headers[name] = value + + response_headers.setdefault("Date", email.utils.formatdate(usegmt=True)) + response_headers.setdefault("Server", USER_AGENT) + + self.write_http_response(http.HTTPStatus.SWITCHING_PROTOCOLS, response_headers) + + self.connection_open() + + return path + + +class WebSocketServer: + """ + WebSocket server returned by :func:`~websockets.server.serve`. + + This class provides the same interface as + :class:`~asyncio.AbstractServer`, namely the + :meth:`~asyncio.AbstractServer.close` and + :meth:`~asyncio.AbstractServer.wait_closed` methods. + + It keeps track of WebSocket connections in order to close them properly + when shutting down. + + Instances of this class store a reference to the :class:`~asyncio.Server` + object returned by :meth:`~asyncio.loop.create_server` rather than inherit + from :class:`~asyncio.Server` in part because + :meth:`~asyncio.loop.create_server` doesn't support passing a custom + :class:`~asyncio.Server` class. + + """ + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + # Store a reference to loop to avoid relying on self.server._loop. + self.loop = loop + + # Keep track of active connections. + self.websockets: Set[WebSocketServerProtocol] = set() + + # Task responsible for closing the server and terminating connections. + self.close_task: Optional[asyncio.Task[None]] = None + + # Completed when the server is closed and connections are terminated. + self.closed_waiter: asyncio.Future[None] = loop.create_future() + + def wrap(self, server: asyncio.AbstractServer) -> None: + """ + Attach to a given :class:`~asyncio.Server`. + + Since :meth:`~asyncio.loop.create_server` doesn't support injecting a + custom ``Server`` class, the easiest solution that doesn't rely on + private :mod:`asyncio` APIs is to: + + - instantiate a :class:`WebSocketServer` + - give the protocol factory a reference to that instance + - call :meth:`~asyncio.loop.create_server` with the factory + - attach the resulting :class:`~asyncio.Server` with this method + + """ + self.server = server + + def register(self, protocol: WebSocketServerProtocol) -> None: + """ + Register a connection with this server. + + """ + self.websockets.add(protocol) + + def unregister(self, protocol: WebSocketServerProtocol) -> None: + """ + Unregister a connection with this server. + + """ + self.websockets.remove(protocol) + + def is_serving(self) -> bool: + """ + Tell whether the server is accepting new connections or shutting down. + + """ + try: + # Python ≥ 3.7 + return self.server.is_serving() + except AttributeError: # pragma: no cover + # Python < 3.7 + return self.server.sockets is not None + + def close(self) -> None: + """ + Close the server. + + This method: + + * closes the underlying :class:`~asyncio.Server`; + * rejects new WebSocket connections with an HTTP 503 (service + unavailable) error; this happens when the server accepted the TCP + connection but didn't complete the WebSocket opening handshake prior + to closing; + * closes open WebSocket connections with close code 1001 (going away). + + :meth:`close` is idempotent. + + """ + if self.close_task is None: + self.close_task = self.loop.create_task(self._close()) + + async def _close(self) -> None: + """ + Implementation of :meth:`close`. + + This calls :meth:`~asyncio.Server.close` on the underlying + :class:`~asyncio.Server` object to stop accepting new connections and + then closes open connections with close code 1001. + + """ + # Stop accepting new connections. + self.server.close() + + # Wait until self.server.close() completes. + await self.server.wait_closed() + + # Wait until all accepted connections reach connection_made() and call + # register(). See https://bugs.python.org/issue34852 for details. + await asyncio.sleep( + 0, loop=self.loop if sys.version_info[:2] < (3, 8) else None + ) + + # Close OPEN connections with status code 1001. Since the server was + # closed, handshake() closes OPENING conections with a HTTP 503 error. + # Wait until all connections are closed. + + # asyncio.wait doesn't accept an empty first argument + if self.websockets: + await asyncio.wait( + [ + asyncio.ensure_future(websocket.close(1001)) + for websocket in self.websockets + ], + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + ) + + # Wait until all connection handlers are complete. + + # asyncio.wait doesn't accept an empty first argument. + if self.websockets: + await asyncio.wait( + [websocket.handler_task for websocket in self.websockets], + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + ) + + # Tell wait_closed() to return. + self.closed_waiter.set_result(None) + + async def wait_closed(self) -> None: + """ + Wait until the server is closed. + + When :meth:`wait_closed` returns, all TCP connections are closed and + all connection handlers have returned. + + """ + await asyncio.shield(self.closed_waiter) + + @property + def sockets(self) -> Optional[List[socket.socket]]: + """ + List of :class:`~socket.socket` objects the server is listening to. + + ``None`` if the server is closed. + + """ + return self.server.sockets + + +class Serve: + """ + + Create, start, and return a WebSocket server on ``host`` and ``port``. + + Whenever a client connects, the server accepts the connection, creates a + :class:`WebSocketServerProtocol`, performs the opening handshake, and + delegates to the connection handler defined by ``ws_handler``. Once the + handler completes, either normally or with an exception, the server + performs the closing handshake and closes the connection. + + Awaiting :func:`serve` yields a :class:`WebSocketServer`. This instance + provides :meth:`~websockets.server.WebSocketServer.close` and + :meth:`~websockets.server.WebSocketServer.wait_closed` methods for + terminating the server and cleaning up its resources. + + When a server is closed with :meth:`~WebSocketServer.close`, it closes all + connections with close code 1001 (going away). Connections handlers, which + are running the ``ws_handler`` coroutine, will receive a + :exc:`~websockets.exceptions.ConnectionClosedOK` exception on their + current or next interaction with the WebSocket connection. + + :func:`serve` can also be used as an asynchronous context manager. In + this case, the server is shut down when exiting the context. + + :func:`serve` is a wrapper around the event loop's + :meth:`~asyncio.loop.create_server` method. It creates and starts a + :class:`~asyncio.Server` with :meth:`~asyncio.loop.create_server`. Then it + wraps the :class:`~asyncio.Server` in a :class:`WebSocketServer` and + returns the :class:`WebSocketServer`. + + The ``ws_handler`` argument is the WebSocket handler. It must be a + coroutine accepting two arguments: a :class:`WebSocketServerProtocol` and + the request URI. + + The ``host`` and ``port`` arguments, as well as unrecognized keyword + arguments, are passed along to :meth:`~asyncio.loop.create_server`. + + For example, you can set the ``ssl`` keyword argument to a + :class:`~ssl.SSLContext` to enable TLS. + + The ``create_protocol`` parameter allows customizing the + :class:`~asyncio.Protocol` that manages the connection. It should be a + callable or class accepting the same arguments as + :class:`WebSocketServerProtocol` and returning an instance of + :class:`WebSocketServerProtocol` or a subclass. It defaults to + :class:`WebSocketServerProtocol`. + + The behavior of ``ping_interval``, ``ping_timeout``, ``close_timeout``, + ``max_size``, ``max_queue``, ``read_limit``, and ``write_limit`` is + described in :class:`~websockets.protocol.WebSocketCommonProtocol`. + + :func:`serve` also accepts the following optional arguments: + + * ``compression`` is a shortcut to configure compression extensions; + by default it enables the "permessage-deflate" extension; set it to + ``None`` to disable compression + * ``origins`` defines acceptable Origin HTTP headers; include ``None`` if + the lack of an origin is acceptable + * ``extensions`` is a list of supported extensions in order of + decreasing preference + * ``subprotocols`` is a list of supported subprotocols in order of + decreasing preference + * ``extra_headers`` sets additional HTTP response headers when the + handshake succeeds; it can be a :class:`~websockets.http.Headers` + instance, a :class:`~collections.abc.Mapping`, an iterable of ``(name, + value)`` pairs, or a callable taking the request path and headers in + arguments and returning one of the above + * ``process_request`` allows intercepting the HTTP request; it must be a + coroutine taking the request path and headers in argument; see + :meth:`~WebSocketServerProtocol.process_request` for details + * ``select_subprotocol`` allows customizing the logic for selecting a + subprotocol; it must be a callable taking the subprotocols offered by + the client and available on the server in argument; see + :meth:`~WebSocketServerProtocol.select_subprotocol` for details + + Since there's no useful way to propagate exceptions triggered in handlers, + they're sent to the ``'websockets.asyncio_server'`` logger instead. + Debugging is much easier if you configure logging to print them:: + + import logging + logger = logging.getLogger("websockets.asyncio_server") + logger.setLevel(logging.ERROR) + logger.addHandler(logging.StreamHandler()) + + """ + + def __init__( + self, + ws_handler: Callable[[WebSocketServerProtocol, str], Awaitable[Any]], + host: Optional[Union[str, Sequence[str]]] = None, + port: Optional[int] = None, + *, + path: Optional[str] = None, + create_protocol: Optional[Type[WebSocketServerProtocol]] = None, + ping_interval: Optional[float] = 20, + ping_timeout: Optional[float] = 20, + close_timeout: Optional[float] = None, + max_size: Optional[int] = 2 ** 20, + max_queue: Optional[int] = 2 ** 5, + read_limit: int = 2 ** 16, + write_limit: int = 2 ** 16, + loop: Optional[asyncio.AbstractEventLoop] = None, + legacy_recv: bool = False, + klass: Optional[Type[WebSocketServerProtocol]] = None, + timeout: Optional[float] = None, + compression: Optional[str] = "deflate", + origins: Optional[Sequence[Optional[Origin]]] = None, + extensions: Optional[Sequence[ServerExtensionFactory]] = None, + subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLikeOrCallable] = None, + process_request: Optional[ + Callable[[str, Headers], Awaitable[Optional[HTTPResponse]]] + ] = None, + select_subprotocol: Optional[ + Callable[[Sequence[Subprotocol], Sequence[Subprotocol]], Subprotocol] + ] = None, + **kwargs: Any, + ) -> None: + # Backwards compatibility: close_timeout used to be called timeout. + if timeout is None: + timeout = 10 + else: + warnings.warn("rename timeout to close_timeout", DeprecationWarning) + # If both are specified, timeout is ignored. + if close_timeout is None: + close_timeout = timeout + + # Backwards compatibility: create_protocol used to be called klass. + if klass is None: + klass = WebSocketServerProtocol + else: + warnings.warn("rename klass to create_protocol", DeprecationWarning) + # If both are specified, klass is ignored. + if create_protocol is None: + create_protocol = klass + + if loop is None: + loop = asyncio.get_event_loop() + + ws_server = WebSocketServer(loop) + + secure = kwargs.get("ssl") is not None + + if compression == "deflate": + if extensions is None: + extensions = [] + if not any( + ext_factory.name == ServerPerMessageDeflateFactory.name + for ext_factory in extensions + ): + extensions = list(extensions) + [ServerPerMessageDeflateFactory()] + elif compression is not None: + raise ValueError(f"unsupported compression: {compression}") + + factory = functools.partial( + create_protocol, + ws_handler, + ws_server, + host=host, + port=port, + secure=secure, + ping_interval=ping_interval, + ping_timeout=ping_timeout, + close_timeout=close_timeout, + max_size=max_size, + max_queue=max_queue, + read_limit=read_limit, + write_limit=write_limit, + loop=loop, + legacy_recv=legacy_recv, + origins=origins, + extensions=extensions, + subprotocols=subprotocols, + extra_headers=extra_headers, + process_request=process_request, + select_subprotocol=select_subprotocol, + ) + + if path is None: + create_server = functools.partial( + loop.create_server, factory, host, port, **kwargs + ) + else: + # unix_serve(path) must not specify host and port parameters. + assert host is None and port is None + create_server = functools.partial( + loop.create_unix_server, factory, path, **kwargs + ) + + # This is a coroutine function. + self._create_server = create_server + self.ws_server = ws_server + + # async with serve(...) + + async def __aenter__(self) -> WebSocketServer: + return await self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + self.ws_server.close() + await self.ws_server.wait_closed() + + # await serve(...) + + def __await__(self) -> Generator[Any, None, WebSocketServer]: + # Create a suitable iterator by calling __await__ on a coroutine. + return self.__await_impl__().__await__() + + async def __await_impl__(self) -> WebSocketServer: + server = await self._create_server() + self.ws_server.wrap(server) + return self.ws_server + + # yield from serve(...) + + __iter__ = __await__ + + +serve = Serve + + +def unix_serve( + ws_handler: Callable[[WebSocketServerProtocol, str], Awaitable[Any]], + path: str, + **kwargs: Any, +) -> Serve: + """ + Similar to :func:`serve`, but for listening on Unix sockets. + + This function calls the event loop's + :meth:`~asyncio.loop.create_unix_server` method. + + It is only available on Unix. + + It's useful for deploying a server behind a reverse proxy such as nginx. + + :param path: file system path to the Unix socket + + """ + return serve(ws_handler, path=path, **kwargs) diff --git a/src/websockets/auth.py b/src/websockets/auth.py index 8198cd9d0..03e8536c5 100644 --- a/src/websockets/auth.py +++ b/src/websockets/auth.py @@ -9,10 +9,10 @@ import http from typing import Any, Awaitable, Callable, Iterable, Optional, Tuple, Type, Union +from .asyncio_server import HTTPResponse, WebSocketServerProtocol from .datastructures import Headers from .exceptions import InvalidHeader from .headers import build_www_authenticate_basic, parse_authorization_basic -from .server import HTTPResponse, WebSocketServerProtocol __all__ = ["BasicAuthWebSocketServerProtocol", "basic_auth_protocol_factory"] diff --git a/src/websockets/client.py b/src/websockets/client.py index f95dae060..c7d153f13 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -1,588 +1,8 @@ -""" -:mod:`websockets.client` defines the WebSocket client APIs. +from .asyncio_client import WebSocketClientProtocol, connect, unix_connect -""" -import asyncio -import collections.abc -import functools -import logging -import warnings -from types import TracebackType -from typing import Any, Generator, List, Optional, Sequence, Tuple, Type, cast - -from .datastructures import Headers, HeadersLike -from .exceptions import ( - InvalidHandshake, - InvalidHeader, - InvalidMessage, - InvalidStatusCode, - NegotiationError, - RedirectHandshake, - SecurityError, -) -from .extensions.base import ClientExtensionFactory, Extension -from .extensions.permessage_deflate import ClientPerMessageDeflateFactory -from .handshake_legacy import build_request, check_response -from .headers import ( - build_authorization_basic, - build_extension, - build_subprotocol, - parse_extension, - parse_subprotocol, -) -from .http import USER_AGENT -from .http_legacy import read_response -from .protocol import WebSocketCommonProtocol -from .typing import ExtensionHeader, Origin, Subprotocol -from .uri import WebSocketURI, parse_uri - - -__all__ = ["connect", "unix_connect", "WebSocketClientProtocol"] - -logger = logging.getLogger(__name__) - - -class WebSocketClientProtocol(WebSocketCommonProtocol): - """ - :class:`~asyncio.Protocol` subclass implementing a WebSocket client. - - This class inherits most of its methods from - :class:`~websockets.protocol.WebSocketCommonProtocol`. - - """ - - is_client = True - side = "client" - - def __init__( - self, - *, - origin: Optional[Origin] = None, - extensions: Optional[Sequence[ClientExtensionFactory]] = None, - subprotocols: Optional[Sequence[Subprotocol]] = None, - extra_headers: Optional[HeadersLike] = None, - **kwargs: Any, - ) -> None: - self.origin = origin - self.available_extensions = extensions - self.available_subprotocols = subprotocols - self.extra_headers = extra_headers - super().__init__(**kwargs) - - def write_http_request(self, path: str, headers: Headers) -> None: - """ - Write request line and headers to the HTTP request. - - """ - self.path = path - self.request_headers = headers - - logger.debug("%s > GET %s HTTP/1.1", self.side, path) - logger.debug("%s > %r", self.side, headers) - - # Since the path and headers only contain ASCII characters, - # we can keep this simple. - request = f"GET {path} HTTP/1.1\r\n" - request += str(headers) - - self.transport.write(request.encode()) - - async def read_http_response(self) -> Tuple[int, Headers]: - """ - Read status line and headers from the HTTP response. - - If the response contains a body, it may be read from ``self.reader`` - after this coroutine returns. - - :raises ~websockets.exceptions.InvalidMessage: if the HTTP message is - malformed or isn't an HTTP/1.1 GET response - - """ - try: - status_code, reason, headers = await read_response(self.reader) - except asyncio.CancelledError: # pragma: no cover - raise - except Exception as exc: - raise InvalidMessage("did not receive a valid HTTP response") from exc - - logger.debug("%s < HTTP/1.1 %d %s", self.side, status_code, reason) - logger.debug("%s < %r", self.side, headers) - - self.response_headers = headers - - return status_code, self.response_headers - - @staticmethod - def process_extensions( - headers: Headers, - available_extensions: Optional[Sequence[ClientExtensionFactory]], - ) -> List[Extension]: - """ - Handle the Sec-WebSocket-Extensions HTTP response header. - - Check that each extension is supported, as well as its parameters. - - Return the list of accepted extensions. - - Raise :exc:`~websockets.exceptions.InvalidHandshake` to abort the - connection. - - :rfc:`6455` leaves the rules up to the specification of each - :extension. - - To provide this level of flexibility, for each extension accepted by - the server, we check for a match with each extension available in the - client configuration. If no match is found, an exception is raised. - - If several variants of the same extension are accepted by the server, - it may be configured several times, which won't make sense in general. - Extensions must implement their own requirements. For this purpose, - the list of previously accepted extensions is provided. - - Other requirements, for example related to mandatory extensions or the - order of extensions, may be implemented by overriding this method. - - """ - accepted_extensions: List[Extension] = [] - - header_values = headers.get_all("Sec-WebSocket-Extensions") - - if header_values: - - if available_extensions is None: - raise InvalidHandshake("no extensions supported") - - parsed_header_values: List[ExtensionHeader] = sum( - [parse_extension(header_value) for header_value in header_values], [] - ) - - for name, response_params in parsed_header_values: - - for extension_factory in available_extensions: - - # Skip non-matching extensions based on their name. - if extension_factory.name != name: - continue - - # Skip non-matching extensions based on their params. - try: - extension = extension_factory.process_response_params( - response_params, accepted_extensions - ) - except NegotiationError: - continue - - # Add matching extension to the final list. - accepted_extensions.append(extension) - - # Break out of the loop once we have a match. - break - - # If we didn't break from the loop, no extension in our list - # matched what the server sent. Fail the connection. - else: - raise NegotiationError( - f"Unsupported extension: " - f"name = {name}, params = {response_params}" - ) - - return accepted_extensions - - @staticmethod - def process_subprotocol( - headers: Headers, available_subprotocols: Optional[Sequence[Subprotocol]] - ) -> Optional[Subprotocol]: - """ - Handle the Sec-WebSocket-Protocol HTTP response header. - - Check that it contains exactly one supported subprotocol. - - Return the selected subprotocol. - - """ - subprotocol: Optional[Subprotocol] = None - - header_values = headers.get_all("Sec-WebSocket-Protocol") - - if header_values: - - if available_subprotocols is None: - raise InvalidHandshake("no subprotocols supported") - - parsed_header_values: Sequence[Subprotocol] = sum( - [parse_subprotocol(header_value) for header_value in header_values], [] - ) - - if len(parsed_header_values) > 1: - subprotocols = ", ".join(parsed_header_values) - raise InvalidHandshake(f"multiple subprotocols: {subprotocols}") - - subprotocol = parsed_header_values[0] - - if subprotocol not in available_subprotocols: - raise NegotiationError(f"unsupported subprotocol: {subprotocol}") - - return subprotocol - - async def handshake( - self, - wsuri: WebSocketURI, - origin: Optional[Origin] = None, - available_extensions: Optional[Sequence[ClientExtensionFactory]] = None, - available_subprotocols: Optional[Sequence[Subprotocol]] = None, - extra_headers: Optional[HeadersLike] = None, - ) -> None: - """ - Perform the client side of the opening handshake. - - :param origin: sets the Origin HTTP header - :param available_extensions: list of supported extensions in the order - in which they should be used - :param available_subprotocols: list of supported subprotocols in order - of decreasing preference - :param extra_headers: sets additional HTTP request headers; it must be - a :class:`~websockets.http.Headers` instance, a - :class:`~collections.abc.Mapping`, or an iterable of ``(name, - value)`` pairs - :raises ~websockets.exceptions.InvalidHandshake: if the handshake - fails - - """ - request_headers = Headers() - - if wsuri.port == (443 if wsuri.secure else 80): # pragma: no cover - request_headers["Host"] = wsuri.host - else: - request_headers["Host"] = f"{wsuri.host}:{wsuri.port}" - - if wsuri.user_info: - request_headers["Authorization"] = build_authorization_basic( - *wsuri.user_info - ) - - if origin is not None: - request_headers["Origin"] = origin - - key = build_request(request_headers) - - if available_extensions is not None: - extensions_header = build_extension( - [ - (extension_factory.name, extension_factory.get_request_params()) - for extension_factory in available_extensions - ] - ) - request_headers["Sec-WebSocket-Extensions"] = extensions_header - - if available_subprotocols is not None: - protocol_header = build_subprotocol(available_subprotocols) - request_headers["Sec-WebSocket-Protocol"] = protocol_header - - if extra_headers is not None: - if isinstance(extra_headers, Headers): - extra_headers = extra_headers.raw_items() - elif isinstance(extra_headers, collections.abc.Mapping): - extra_headers = extra_headers.items() - for name, value in extra_headers: - request_headers[name] = value - - request_headers.setdefault("User-Agent", USER_AGENT) - - self.write_http_request(wsuri.resource_name, request_headers) - - status_code, response_headers = await self.read_http_response() - if status_code in (301, 302, 303, 307, 308): - if "Location" not in response_headers: - raise InvalidHeader("Location") - raise RedirectHandshake(response_headers["Location"]) - elif status_code != 101: - raise InvalidStatusCode(status_code) - - check_response(response_headers, key) - - self.extensions = self.process_extensions( - response_headers, available_extensions - ) - - self.subprotocol = self.process_subprotocol( - response_headers, available_subprotocols - ) - - self.connection_open() - - -class Connect: - """ - Connect to the WebSocket server at the given ``uri``. - - Awaiting :func:`connect` yields a :class:`WebSocketClientProtocol` which - can then be used to send and receive messages. - - :func:`connect` can also be used as a asynchronous context manager. In - that case, the connection is closed when exiting the context. - - :func:`connect` is a wrapper around the event loop's - :meth:`~asyncio.loop.create_connection` method. Unknown keyword arguments - are passed to :meth:`~asyncio.loop.create_connection`. - - For example, you can set the ``ssl`` keyword argument to a - :class:`~ssl.SSLContext` to enforce some TLS settings. When connecting to - a ``wss://`` URI, if this argument isn't provided explicitly, - :func:`ssl.create_default_context` is called to create a context. - - You can connect to a different host and port from those found in ``uri`` - by setting ``host`` and ``port`` keyword arguments. This only changes the - destination of the TCP connection. The host name from ``uri`` is still - used in the TLS handshake for secure connections and in the ``Host`` HTTP - header. - - The ``create_protocol`` parameter allows customizing the - :class:`~asyncio.Protocol` that manages the connection. It should be a - callable or class accepting the same arguments as - :class:`WebSocketClientProtocol` and returning an instance of - :class:`WebSocketClientProtocol` or a subclass. It defaults to - :class:`WebSocketClientProtocol`. - - The behavior of ``ping_interval``, ``ping_timeout``, ``close_timeout``, - ``max_size``, ``max_queue``, ``read_limit``, and ``write_limit`` is - described in :class:`~websockets.protocol.WebSocketCommonProtocol`. - - :func:`connect` also accepts the following optional arguments: - - * ``compression`` is a shortcut to configure compression extensions; - by default it enables the "permessage-deflate" extension; set it to - ``None`` to disable compression - * ``origin`` sets the Origin HTTP header - * ``extensions`` is a list of supported extensions in order of - decreasing preference - * ``subprotocols`` is a list of supported subprotocols in order of - decreasing preference - * ``extra_headers`` sets additional HTTP request headers; it can be a - :class:`~websockets.http.Headers` instance, a - :class:`~collections.abc.Mapping`, or an iterable of ``(name, value)`` - pairs - - :raises ~websockets.uri.InvalidURI: if ``uri`` is invalid - :raises ~websockets.handshake.InvalidHandshake: if the opening handshake - fails - - """ - - MAX_REDIRECTS_ALLOWED = 10 - - def __init__( - self, - uri: str, - *, - path: Optional[str] = None, - create_protocol: Optional[Type[WebSocketClientProtocol]] = None, - ping_interval: Optional[float] = 20, - ping_timeout: Optional[float] = 20, - close_timeout: Optional[float] = None, - max_size: Optional[int] = 2 ** 20, - max_queue: Optional[int] = 2 ** 5, - read_limit: int = 2 ** 16, - write_limit: int = 2 ** 16, - loop: Optional[asyncio.AbstractEventLoop] = None, - legacy_recv: bool = False, - klass: Optional[Type[WebSocketClientProtocol]] = None, - timeout: Optional[float] = None, - compression: Optional[str] = "deflate", - origin: Optional[Origin] = None, - extensions: Optional[Sequence[ClientExtensionFactory]] = None, - subprotocols: Optional[Sequence[Subprotocol]] = None, - extra_headers: Optional[HeadersLike] = None, - **kwargs: Any, - ) -> None: - # Backwards compatibility: close_timeout used to be called timeout. - if timeout is None: - timeout = 10 - else: - warnings.warn("rename timeout to close_timeout", DeprecationWarning) - # If both are specified, timeout is ignored. - if close_timeout is None: - close_timeout = timeout - - # Backwards compatibility: create_protocol used to be called klass. - if klass is None: - klass = WebSocketClientProtocol - else: - warnings.warn("rename klass to create_protocol", DeprecationWarning) - # If both are specified, klass is ignored. - if create_protocol is None: - create_protocol = klass - - if loop is None: - loop = asyncio.get_event_loop() - - wsuri = parse_uri(uri) - if wsuri.secure: - kwargs.setdefault("ssl", True) - elif kwargs.get("ssl") is not None: - raise ValueError( - "connect() received a ssl argument for a ws:// URI, " - "use a wss:// URI to enable TLS" - ) - - if compression == "deflate": - if extensions is None: - extensions = [] - if not any( - extension_factory.name == ClientPerMessageDeflateFactory.name - for extension_factory in extensions - ): - extensions = list(extensions) + [ - ClientPerMessageDeflateFactory(client_max_window_bits=True) - ] - elif compression is not None: - raise ValueError(f"unsupported compression: {compression}") - - factory = functools.partial( - create_protocol, - ping_interval=ping_interval, - ping_timeout=ping_timeout, - close_timeout=close_timeout, - max_size=max_size, - max_queue=max_queue, - read_limit=read_limit, - write_limit=write_limit, - loop=loop, - host=wsuri.host, - port=wsuri.port, - secure=wsuri.secure, - legacy_recv=legacy_recv, - origin=origin, - extensions=extensions, - subprotocols=subprotocols, - extra_headers=extra_headers, - ) - - if path is None: - host: Optional[str] - port: Optional[int] - if kwargs.get("sock") is None: - host, port = wsuri.host, wsuri.port - else: - # If sock is given, host and port shouldn't be specified. - host, port = None, None - # If host and port are given, override values from the URI. - host = kwargs.pop("host", host) - port = kwargs.pop("port", port) - create_connection = functools.partial( - loop.create_connection, factory, host, port, **kwargs - ) - else: - create_connection = functools.partial( - loop.create_unix_connection, factory, path, **kwargs - ) - - # This is a coroutine function. - self._create_connection = create_connection - self._wsuri = wsuri - - def handle_redirect(self, uri: str) -> None: - # Update the state of this instance to connect to a new URI. - old_wsuri = self._wsuri - new_wsuri = parse_uri(uri) - - # Forbid TLS downgrade. - if old_wsuri.secure and not new_wsuri.secure: - raise SecurityError("redirect from WSS to WS") - - same_origin = ( - old_wsuri.host == new_wsuri.host and old_wsuri.port == new_wsuri.port - ) - - # Rewrite the host and port arguments for cross-origin redirects. - # This preserves connection overrides with the host and port - # arguments if the redirect points to the same host and port. - if not same_origin: - # Replace the host and port argument passed to the protocol factory. - factory = self._create_connection.args[0] - factory = functools.partial( - factory.func, - *factory.args, - **dict(factory.keywords, host=new_wsuri.host, port=new_wsuri.port), - ) - # Replace the host and port argument passed to create_connection. - self._create_connection = functools.partial( - self._create_connection.func, - *(factory, new_wsuri.host, new_wsuri.port), - **self._create_connection.keywords, - ) - - # Set the new WebSocket URI. This suffices for same-origin redirects. - self._wsuri = new_wsuri - - # async with connect(...) - - async def __aenter__(self) -> WebSocketClientProtocol: - return await self - - async def __aexit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> None: - await self.ws_client.close() - - # await connect(...) - - def __await__(self) -> Generator[Any, None, WebSocketClientProtocol]: - # Create a suitable iterator by calling __await__ on a coroutine. - return self.__await_impl__().__await__() - - async def __await_impl__(self) -> WebSocketClientProtocol: - for redirects in range(self.MAX_REDIRECTS_ALLOWED): - transport, protocol = await self._create_connection() - # https://github.com/python/typeshed/pull/2756 - transport = cast(asyncio.Transport, transport) - protocol = cast(WebSocketClientProtocol, protocol) - - try: - try: - await protocol.handshake( - self._wsuri, - origin=protocol.origin, - available_extensions=protocol.available_extensions, - available_subprotocols=protocol.available_subprotocols, - extra_headers=protocol.extra_headers, - ) - except Exception: - protocol.fail_connection() - await protocol.wait_closed() - raise - else: - self.ws_client = protocol - return protocol - except RedirectHandshake as exc: - self.handle_redirect(exc.uri) - else: - raise SecurityError("too many redirects") - - # yield from connect(...) - - __iter__ = __await__ - - -connect = Connect - - -def unix_connect(path: str, uri: str = "ws://localhost/", **kwargs: Any) -> Connect: - """ - Similar to :func:`connect`, but for connecting to a Unix socket. - - This function calls the event loop's - :meth:`~asyncio.loop.create_unix_connection` method. - - It is only available on Unix. - - It's mainly useful for debugging servers listening on Unix sockets. - - :param path: file system path to the Unix socket - :param uri: WebSocket URI - - """ - return connect(uri=uri, path=path, **kwargs) +__all__ = [ + "connect", + "unix_connect", + "WebSocketClientProtocol", +] diff --git a/src/websockets/server.py b/src/websockets/server.py index 522c76114..ec94a2fbf 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -1,1004 +1,9 @@ -""" -:mod:`websockets.server` defines the WebSocket server APIs. +from .asyncio_server import WebSocketServer, WebSocketServerProtocol, serve, unix_serve -""" -import asyncio -import collections.abc -import email.utils -import functools -import http -import logging -import socket -import sys -import warnings -from types import TracebackType -from typing import ( - Any, - Awaitable, - Callable, - Generator, - List, - Optional, - Sequence, - Set, - Tuple, - Type, - Union, - cast, -) - -from .datastructures import Headers, HeadersLike, MultipleValuesError -from .exceptions import ( - AbortHandshake, - InvalidHandshake, - InvalidHeader, - InvalidMessage, - InvalidOrigin, - InvalidUpgrade, - NegotiationError, -) -from .extensions.base import Extension, ServerExtensionFactory -from .extensions.permessage_deflate import ServerPerMessageDeflateFactory -from .handshake_legacy import build_response, check_request -from .headers import build_extension, parse_extension, parse_subprotocol -from .http import USER_AGENT -from .http_legacy import read_request -from .protocol import WebSocketCommonProtocol -from .typing import ExtensionHeader, Origin, Subprotocol - - -__all__ = ["serve", "unix_serve", "WebSocketServerProtocol", "WebSocketServer"] - -logger = logging.getLogger(__name__) - - -HeadersLikeOrCallable = Union[HeadersLike, Callable[[str, Headers], HeadersLike]] - -HTTPResponse = Tuple[http.HTTPStatus, HeadersLike, bytes] - - -class WebSocketServerProtocol(WebSocketCommonProtocol): - """ - :class:`~asyncio.Protocol` subclass implementing a WebSocket server. - - This class inherits most of its methods from - :class:`~websockets.protocol.WebSocketCommonProtocol`. - - For the sake of simplicity, it doesn't rely on a full HTTP implementation. - Its support for HTTP responses is very limited. - - """ - - is_client = False - side = "server" - - def __init__( - self, - ws_handler: Callable[["WebSocketServerProtocol", str], Awaitable[Any]], - ws_server: "WebSocketServer", - *, - origins: Optional[Sequence[Optional[Origin]]] = None, - extensions: Optional[Sequence[ServerExtensionFactory]] = None, - subprotocols: Optional[Sequence[Subprotocol]] = None, - extra_headers: Optional[HeadersLikeOrCallable] = None, - process_request: Optional[ - Callable[[str, Headers], Awaitable[Optional[HTTPResponse]]] - ] = None, - select_subprotocol: Optional[ - Callable[[Sequence[Subprotocol], Sequence[Subprotocol]], Subprotocol] - ] = None, - **kwargs: Any, - ) -> None: - # For backwards compatibility with 6.0 or earlier. - if origins is not None and "" in origins: - warnings.warn("use None instead of '' in origins", DeprecationWarning) - origins = [None if origin == "" else origin for origin in origins] - self.ws_handler = ws_handler - self.ws_server = ws_server - self.origins = origins - self.available_extensions = extensions - self.available_subprotocols = subprotocols - self.extra_headers = extra_headers - self._process_request = process_request - self._select_subprotocol = select_subprotocol - super().__init__(**kwargs) - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - """ - Register connection and initialize a task to handle it. - - """ - super().connection_made(transport) - # Register the connection with the server before creating the handler - # task. Registering at the beginning of the handler coroutine would - # create a race condition between the creation of the task, which - # schedules its execution, and the moment the handler starts running. - self.ws_server.register(self) - self.handler_task = self.loop.create_task(self.handler()) - - async def handler(self) -> None: - """ - Handle the lifecycle of a WebSocket connection. - - Since this method doesn't have a caller able to handle exceptions, it - attemps to log relevant ones and guarantees that the TCP connection is - closed before exiting. - - """ - try: - - try: - path = await self.handshake( - origins=self.origins, - available_extensions=self.available_extensions, - available_subprotocols=self.available_subprotocols, - extra_headers=self.extra_headers, - ) - except asyncio.CancelledError: # pragma: no cover - raise - except ConnectionError: - logger.debug("Connection error in opening handshake", exc_info=True) - raise - except Exception as exc: - if isinstance(exc, AbortHandshake): - status, headers, body = exc.status, exc.headers, exc.body - elif isinstance(exc, InvalidOrigin): - logger.debug("Invalid origin", exc_info=True) - status, headers, body = ( - http.HTTPStatus.FORBIDDEN, - Headers(), - f"Failed to open a WebSocket connection: {exc}.\n".encode(), - ) - elif isinstance(exc, InvalidUpgrade): - logger.debug("Invalid upgrade", exc_info=True) - status, headers, body = ( - http.HTTPStatus.UPGRADE_REQUIRED, - Headers([("Upgrade", "websocket")]), - ( - f"Failed to open a WebSocket connection: {exc}.\n" - f"\n" - f"You cannot access a WebSocket server directly " - f"with a browser. You need a WebSocket client.\n" - ).encode(), - ) - elif isinstance(exc, InvalidHandshake): - logger.debug("Invalid handshake", exc_info=True) - status, headers, body = ( - http.HTTPStatus.BAD_REQUEST, - Headers(), - f"Failed to open a WebSocket connection: {exc}.\n".encode(), - ) - else: - logger.warning("Error in opening handshake", exc_info=True) - status, headers, body = ( - http.HTTPStatus.INTERNAL_SERVER_ERROR, - Headers(), - ( - b"Failed to open a WebSocket connection.\n" - b"See server log for more information.\n" - ), - ) - - headers.setdefault("Date", email.utils.formatdate(usegmt=True)) - headers.setdefault("Server", USER_AGENT) - headers.setdefault("Content-Length", str(len(body))) - headers.setdefault("Content-Type", "text/plain") - headers.setdefault("Connection", "close") - - self.write_http_response(status, headers, body) - self.fail_connection() - await self.wait_closed() - return - - try: - await self.ws_handler(self, path) - except Exception: - logger.error("Error in connection handler", exc_info=True) - if not self.closed: - self.fail_connection(1011) - raise - - try: - await self.close() - except ConnectionError: - logger.debug("Connection error in closing handshake", exc_info=True) - raise - except Exception: - logger.warning("Error in closing handshake", exc_info=True) - raise - - except Exception: - # Last-ditch attempt to avoid leaking connections on errors. - try: - self.transport.close() - except Exception: # pragma: no cover - pass - - finally: - # Unregister the connection with the server when the handler task - # terminates. Registration is tied to the lifecycle of the handler - # task because the server waits for tasks attached to registered - # connections before terminating. - self.ws_server.unregister(self) - - async def read_http_request(self) -> Tuple[str, Headers]: - """ - Read request line and headers from the HTTP request. - - If the request contains a body, it may be read from ``self.reader`` - after this coroutine returns. - - :raises ~websockets.exceptions.InvalidMessage: if the HTTP message is - malformed or isn't an HTTP/1.1 GET request - - """ - try: - path, headers = await read_request(self.reader) - except asyncio.CancelledError: # pragma: no cover - raise - except Exception as exc: - raise InvalidMessage("did not receive a valid HTTP request") from exc - - logger.debug("%s < GET %s HTTP/1.1", self.side, path) - logger.debug("%s < %r", self.side, headers) - - self.path = path - self.request_headers = headers - - return path, headers - - def write_http_response( - self, status: http.HTTPStatus, headers: Headers, body: Optional[bytes] = None - ) -> None: - """ - Write status line and headers to the HTTP response. - - This coroutine is also able to write a response body. - - """ - self.response_headers = headers - - logger.debug("%s > HTTP/1.1 %d %s", self.side, status.value, status.phrase) - logger.debug("%s > %r", self.side, headers) - - # Since the status line and headers only contain ASCII characters, - # we can keep this simple. - response = f"HTTP/1.1 {status.value} {status.phrase}\r\n" - response += str(headers) - - self.transport.write(response.encode()) - - if body is not None: - logger.debug("%s > body (%d bytes)", self.side, len(body)) - self.transport.write(body) - - async def process_request( - self, path: str, request_headers: Headers - ) -> Optional[HTTPResponse]: - """ - Intercept the HTTP request and return an HTTP response if appropriate. - - If ``process_request`` returns ``None``, the WebSocket handshake - continues. If it returns 3-uple containing a status code, response - headers and a response body, that HTTP response is sent and the - connection is closed. In that case: - - * The HTTP status must be a :class:`~http.HTTPStatus`. - * HTTP headers must be a :class:`~websockets.http.Headers` instance, a - :class:`~collections.abc.Mapping`, or an iterable of ``(name, - value)`` pairs. - * The HTTP response body must be :class:`bytes`. It may be empty. - - This coroutine may be overridden in a :class:`WebSocketServerProtocol` - subclass, for example: - - * to return a HTTP 200 OK response on a given path; then a load - balancer can use this path for a health check; - * to authenticate the request and return a HTTP 401 Unauthorized or a - HTTP 403 Forbidden when authentication fails. - - Instead of subclassing, it is possible to override this method by - passing a ``process_request`` argument to the :func:`serve` function - or the :class:`WebSocketServerProtocol` constructor. This is - equivalent, except ``process_request`` won't have access to the - protocol instance, so it can't store information for later use. - - ``process_request`` is expected to complete quickly. If it may run for - a long time, then it should await :meth:`wait_closed` and exit if - :meth:`wait_closed` completes, or else it could prevent the server - from shutting down. - - :param path: request path, including optional query string - :param request_headers: request headers - - """ - if self._process_request is not None: - response = self._process_request(path, request_headers) - if isinstance(response, Awaitable): - return await response - else: - # For backwards compatibility with 7.0. - warnings.warn( - "declare process_request as a coroutine", DeprecationWarning - ) - return response # type: ignore - return None - - @staticmethod - def process_origin( - headers: Headers, origins: Optional[Sequence[Optional[Origin]]] = None - ) -> Optional[Origin]: - """ - Handle the Origin HTTP request header. - - :param headers: request headers - :param origins: optional list of acceptable origins - :raises ~websockets.exceptions.InvalidOrigin: if the origin isn't - acceptable - - """ - # "The user agent MUST NOT include more than one Origin header field" - # per https://tools.ietf.org/html/rfc6454#section-7.3. - try: - origin = cast(Origin, headers.get("Origin")) - except MultipleValuesError as exc: - raise InvalidHeader("Origin", "more than one Origin header found") from exc - if origins is not None: - if origin not in origins: - raise InvalidOrigin(origin) - return origin - - @staticmethod - def process_extensions( - headers: Headers, - available_extensions: Optional[Sequence[ServerExtensionFactory]], - ) -> Tuple[Optional[str], List[Extension]]: - """ - Handle the Sec-WebSocket-Extensions HTTP request header. - - Accept or reject each extension proposed in the client request. - Negotiate parameters for accepted extensions. - - Return the Sec-WebSocket-Extensions HTTP response header and the list - of accepted extensions. - - :rfc:`6455` leaves the rules up to the specification of each - :extension. - - To provide this level of flexibility, for each extension proposed by - the client, we check for a match with each extension available in the - server configuration. If no match is found, the extension is ignored. - - If several variants of the same extension are proposed by the client, - it may be accepted several times, which won't make sense in general. - Extensions must implement their own requirements. For this purpose, - the list of previously accepted extensions is provided. - - This process doesn't allow the server to reorder extensions. It can - only select a subset of the extensions proposed by the client. - - Other requirements, for example related to mandatory extensions or the - order of extensions, may be implemented by overriding this method. - - :param headers: request headers - :param extensions: optional list of supported extensions - :raises ~websockets.exceptions.InvalidHandshake: to abort the - handshake with an HTTP 400 error code - - """ - response_header_value: Optional[str] = None - - extension_headers: List[ExtensionHeader] = [] - accepted_extensions: List[Extension] = [] - - header_values = headers.get_all("Sec-WebSocket-Extensions") - - if header_values and available_extensions: - - parsed_header_values: List[ExtensionHeader] = sum( - [parse_extension(header_value) for header_value in header_values], [] - ) - - for name, request_params in parsed_header_values: - - for ext_factory in available_extensions: - - # Skip non-matching extensions based on their name. - if ext_factory.name != name: - continue - - # Skip non-matching extensions based on their params. - try: - response_params, extension = ext_factory.process_request_params( - request_params, accepted_extensions - ) - except NegotiationError: - continue - - # Add matching extension to the final list. - extension_headers.append((name, response_params)) - accepted_extensions.append(extension) - - # Break out of the loop once we have a match. - break - - # If we didn't break from the loop, no extension in our list - # matched what the client sent. The extension is declined. - - # Serialize extension header. - if extension_headers: - response_header_value = build_extension(extension_headers) - - return response_header_value, accepted_extensions - - # Not @staticmethod because it calls self.select_subprotocol() - def process_subprotocol( - self, headers: Headers, available_subprotocols: Optional[Sequence[Subprotocol]] - ) -> Optional[Subprotocol]: - """ - Handle the Sec-WebSocket-Protocol HTTP request header. - - Return Sec-WebSocket-Protocol HTTP response header, which is the same - as the selected subprotocol. - - :param headers: request headers - :param available_subprotocols: optional list of supported subprotocols - :raises ~websockets.exceptions.InvalidHandshake: to abort the - handshake with an HTTP 400 error code - - """ - subprotocol: Optional[Subprotocol] = None - - header_values = headers.get_all("Sec-WebSocket-Protocol") - - if header_values and available_subprotocols: - - parsed_header_values: List[Subprotocol] = sum( - [parse_subprotocol(header_value) for header_value in header_values], [] - ) - - subprotocol = self.select_subprotocol( - parsed_header_values, available_subprotocols - ) - - return subprotocol - - def select_subprotocol( - self, - client_subprotocols: Sequence[Subprotocol], - server_subprotocols: Sequence[Subprotocol], - ) -> Optional[Subprotocol]: - """ - Pick a subprotocol among those offered by the client. - - If several subprotocols are supported by the client and the server, - the default implementation selects the preferred subprotocols by - giving equal value to the priorities of the client and the server. - - If no subprotocol is supported by the client and the server, it - proceeds without a subprotocol. - - This is unlikely to be the most useful implementation in practice, as - many servers providing a subprotocol will require that the client uses - that subprotocol. Such rules can be implemented in a subclass. - - Instead of subclassing, it is possible to override this method by - passing a ``select_subprotocol`` argument to the :func:`serve` - function or the :class:`WebSocketServerProtocol` constructor - - :param client_subprotocols: list of subprotocols offered by the client - :param server_subprotocols: list of subprotocols available on the server - - """ - if self._select_subprotocol is not None: - return self._select_subprotocol(client_subprotocols, server_subprotocols) - - subprotocols = set(client_subprotocols) & set(server_subprotocols) - if not subprotocols: - return None - priority = lambda p: ( - client_subprotocols.index(p) + server_subprotocols.index(p) - ) - return sorted(subprotocols, key=priority)[0] - - async def handshake( - self, - origins: Optional[Sequence[Optional[Origin]]] = None, - available_extensions: Optional[Sequence[ServerExtensionFactory]] = None, - available_subprotocols: Optional[Sequence[Subprotocol]] = None, - extra_headers: Optional[HeadersLikeOrCallable] = None, - ) -> str: - """ - Perform the server side of the opening handshake. - - Return the path of the URI of the request. - - :param origins: list of acceptable values of the Origin HTTP header; - include ``None`` if the lack of an origin is acceptable - :param available_extensions: list of supported extensions in the order - in which they should be used - :param available_subprotocols: list of supported subprotocols in order - of decreasing preference - :param extra_headers: sets additional HTTP response headers when the - handshake succeeds; it can be a :class:`~websockets.http.Headers` - instance, a :class:`~collections.abc.Mapping`, an iterable of - ``(name, value)`` pairs, or a callable taking the request path and - headers in arguments and returning one of the above. - :raises ~websockets.exceptions.InvalidHandshake: if the handshake - fails - - """ - path, request_headers = await self.read_http_request() - - # Hook for customizing request handling, for example checking - # authentication or treating some paths as plain HTTP endpoints. - early_response_awaitable = self.process_request(path, request_headers) - if isinstance(early_response_awaitable, Awaitable): - early_response = await early_response_awaitable - else: - # For backwards compatibility with 7.0. - warnings.warn("declare process_request as a coroutine", DeprecationWarning) - early_response = early_response_awaitable # type: ignore - - # Change the response to a 503 error if the server is shutting down. - if not self.ws_server.is_serving(): - early_response = ( - http.HTTPStatus.SERVICE_UNAVAILABLE, - [], - b"Server is shutting down.\n", - ) - - if early_response is not None: - raise AbortHandshake(*early_response) - - key = check_request(request_headers) - - self.origin = self.process_origin(request_headers, origins) - - extensions_header, self.extensions = self.process_extensions( - request_headers, available_extensions - ) - - protocol_header = self.subprotocol = self.process_subprotocol( - request_headers, available_subprotocols - ) - - response_headers = Headers() - - build_response(response_headers, key) - - if extensions_header is not None: - response_headers["Sec-WebSocket-Extensions"] = extensions_header - - if protocol_header is not None: - response_headers["Sec-WebSocket-Protocol"] = protocol_header - - if callable(extra_headers): - extra_headers = extra_headers(path, self.request_headers) - if extra_headers is not None: - if isinstance(extra_headers, Headers): - extra_headers = extra_headers.raw_items() - elif isinstance(extra_headers, collections.abc.Mapping): - extra_headers = extra_headers.items() - for name, value in extra_headers: - response_headers[name] = value - - response_headers.setdefault("Date", email.utils.formatdate(usegmt=True)) - response_headers.setdefault("Server", USER_AGENT) - - self.write_http_response(http.HTTPStatus.SWITCHING_PROTOCOLS, response_headers) - - self.connection_open() - - return path - - -class WebSocketServer: - """ - WebSocket server returned by :func:`~websockets.server.serve`. - - This class provides the same interface as - :class:`~asyncio.AbstractServer`, namely the - :meth:`~asyncio.AbstractServer.close` and - :meth:`~asyncio.AbstractServer.wait_closed` methods. - - It keeps track of WebSocket connections in order to close them properly - when shutting down. - - Instances of this class store a reference to the :class:`~asyncio.Server` - object returned by :meth:`~asyncio.loop.create_server` rather than inherit - from :class:`~asyncio.Server` in part because - :meth:`~asyncio.loop.create_server` doesn't support passing a custom - :class:`~asyncio.Server` class. - - """ - - def __init__(self, loop: asyncio.AbstractEventLoop) -> None: - # Store a reference to loop to avoid relying on self.server._loop. - self.loop = loop - - # Keep track of active connections. - self.websockets: Set[WebSocketServerProtocol] = set() - - # Task responsible for closing the server and terminating connections. - self.close_task: Optional[asyncio.Task[None]] = None - - # Completed when the server is closed and connections are terminated. - self.closed_waiter: asyncio.Future[None] = loop.create_future() - - def wrap(self, server: asyncio.AbstractServer) -> None: - """ - Attach to a given :class:`~asyncio.Server`. - - Since :meth:`~asyncio.loop.create_server` doesn't support injecting a - custom ``Server`` class, the easiest solution that doesn't rely on - private :mod:`asyncio` APIs is to: - - - instantiate a :class:`WebSocketServer` - - give the protocol factory a reference to that instance - - call :meth:`~asyncio.loop.create_server` with the factory - - attach the resulting :class:`~asyncio.Server` with this method - - """ - self.server = server - - def register(self, protocol: WebSocketServerProtocol) -> None: - """ - Register a connection with this server. - - """ - self.websockets.add(protocol) - - def unregister(self, protocol: WebSocketServerProtocol) -> None: - """ - Unregister a connection with this server. - - """ - self.websockets.remove(protocol) - - def is_serving(self) -> bool: - """ - Tell whether the server is accepting new connections or shutting down. - - """ - try: - # Python ≥ 3.7 - return self.server.is_serving() - except AttributeError: # pragma: no cover - # Python < 3.7 - return self.server.sockets is not None - - def close(self) -> None: - """ - Close the server. - - This method: - - * closes the underlying :class:`~asyncio.Server`; - * rejects new WebSocket connections with an HTTP 503 (service - unavailable) error; this happens when the server accepted the TCP - connection but didn't complete the WebSocket opening handshake prior - to closing; - * closes open WebSocket connections with close code 1001 (going away). - - :meth:`close` is idempotent. - - """ - if self.close_task is None: - self.close_task = self.loop.create_task(self._close()) - - async def _close(self) -> None: - """ - Implementation of :meth:`close`. - - This calls :meth:`~asyncio.Server.close` on the underlying - :class:`~asyncio.Server` object to stop accepting new connections and - then closes open connections with close code 1001. - - """ - # Stop accepting new connections. - self.server.close() - - # Wait until self.server.close() completes. - await self.server.wait_closed() - - # Wait until all accepted connections reach connection_made() and call - # register(). See https://bugs.python.org/issue34852 for details. - await asyncio.sleep( - 0, loop=self.loop if sys.version_info[:2] < (3, 8) else None - ) - - # Close OPEN connections with status code 1001. Since the server was - # closed, handshake() closes OPENING conections with a HTTP 503 error. - # Wait until all connections are closed. - - # asyncio.wait doesn't accept an empty first argument - if self.websockets: - await asyncio.wait( - [ - asyncio.ensure_future(websocket.close(1001)) - for websocket in self.websockets - ], - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - ) - - # Wait until all connection handlers are complete. - - # asyncio.wait doesn't accept an empty first argument. - if self.websockets: - await asyncio.wait( - [websocket.handler_task for websocket in self.websockets], - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - ) - - # Tell wait_closed() to return. - self.closed_waiter.set_result(None) - - async def wait_closed(self) -> None: - """ - Wait until the server is closed. - - When :meth:`wait_closed` returns, all TCP connections are closed and - all connection handlers have returned. - - """ - await asyncio.shield(self.closed_waiter) - - @property - def sockets(self) -> Optional[List[socket.socket]]: - """ - List of :class:`~socket.socket` objects the server is listening to. - - ``None`` if the server is closed. - - """ - return self.server.sockets - - -class Serve: - """ - - Create, start, and return a WebSocket server on ``host`` and ``port``. - - Whenever a client connects, the server accepts the connection, creates a - :class:`WebSocketServerProtocol`, performs the opening handshake, and - delegates to the connection handler defined by ``ws_handler``. Once the - handler completes, either normally or with an exception, the server - performs the closing handshake and closes the connection. - - Awaiting :func:`serve` yields a :class:`WebSocketServer`. This instance - provides :meth:`~websockets.server.WebSocketServer.close` and - :meth:`~websockets.server.WebSocketServer.wait_closed` methods for - terminating the server and cleaning up its resources. - - When a server is closed with :meth:`~WebSocketServer.close`, it closes all - connections with close code 1001 (going away). Connections handlers, which - are running the ``ws_handler`` coroutine, will receive a - :exc:`~websockets.exceptions.ConnectionClosedOK` exception on their - current or next interaction with the WebSocket connection. - - :func:`serve` can also be used as an asynchronous context manager. In - this case, the server is shut down when exiting the context. - - :func:`serve` is a wrapper around the event loop's - :meth:`~asyncio.loop.create_server` method. It creates and starts a - :class:`~asyncio.Server` with :meth:`~asyncio.loop.create_server`. Then it - wraps the :class:`~asyncio.Server` in a :class:`WebSocketServer` and - returns the :class:`WebSocketServer`. - - The ``ws_handler`` argument is the WebSocket handler. It must be a - coroutine accepting two arguments: a :class:`WebSocketServerProtocol` and - the request URI. - - The ``host`` and ``port`` arguments, as well as unrecognized keyword - arguments, are passed along to :meth:`~asyncio.loop.create_server`. - - For example, you can set the ``ssl`` keyword argument to a - :class:`~ssl.SSLContext` to enable TLS. - - The ``create_protocol`` parameter allows customizing the - :class:`~asyncio.Protocol` that manages the connection. It should be a - callable or class accepting the same arguments as - :class:`WebSocketServerProtocol` and returning an instance of - :class:`WebSocketServerProtocol` or a subclass. It defaults to - :class:`WebSocketServerProtocol`. - - The behavior of ``ping_interval``, ``ping_timeout``, ``close_timeout``, - ``max_size``, ``max_queue``, ``read_limit``, and ``write_limit`` is - described in :class:`~websockets.protocol.WebSocketCommonProtocol`. - - :func:`serve` also accepts the following optional arguments: - - * ``compression`` is a shortcut to configure compression extensions; - by default it enables the "permessage-deflate" extension; set it to - ``None`` to disable compression - * ``origins`` defines acceptable Origin HTTP headers; include ``None`` if - the lack of an origin is acceptable - * ``extensions`` is a list of supported extensions in order of - decreasing preference - * ``subprotocols`` is a list of supported subprotocols in order of - decreasing preference - * ``extra_headers`` sets additional HTTP response headers when the - handshake succeeds; it can be a :class:`~websockets.http.Headers` - instance, a :class:`~collections.abc.Mapping`, an iterable of ``(name, - value)`` pairs, or a callable taking the request path and headers in - arguments and returning one of the above - * ``process_request`` allows intercepting the HTTP request; it must be a - coroutine taking the request path and headers in argument; see - :meth:`~WebSocketServerProtocol.process_request` for details - * ``select_subprotocol`` allows customizing the logic for selecting a - subprotocol; it must be a callable taking the subprotocols offered by - the client and available on the server in argument; see - :meth:`~WebSocketServerProtocol.select_subprotocol` for details - - Since there's no useful way to propagate exceptions triggered in handlers, - they're sent to the ``'websockets.server'`` logger instead. Debugging is - much easier if you configure logging to print them:: - - import logging - logger = logging.getLogger('websockets.server') - logger.setLevel(logging.ERROR) - logger.addHandler(logging.StreamHandler()) - - """ - - def __init__( - self, - ws_handler: Callable[[WebSocketServerProtocol, str], Awaitable[Any]], - host: Optional[Union[str, Sequence[str]]] = None, - port: Optional[int] = None, - *, - path: Optional[str] = None, - create_protocol: Optional[Type[WebSocketServerProtocol]] = None, - ping_interval: Optional[float] = 20, - ping_timeout: Optional[float] = 20, - close_timeout: Optional[float] = None, - max_size: Optional[int] = 2 ** 20, - max_queue: Optional[int] = 2 ** 5, - read_limit: int = 2 ** 16, - write_limit: int = 2 ** 16, - loop: Optional[asyncio.AbstractEventLoop] = None, - legacy_recv: bool = False, - klass: Optional[Type[WebSocketServerProtocol]] = None, - timeout: Optional[float] = None, - compression: Optional[str] = "deflate", - origins: Optional[Sequence[Optional[Origin]]] = None, - extensions: Optional[Sequence[ServerExtensionFactory]] = None, - subprotocols: Optional[Sequence[Subprotocol]] = None, - extra_headers: Optional[HeadersLikeOrCallable] = None, - process_request: Optional[ - Callable[[str, Headers], Awaitable[Optional[HTTPResponse]]] - ] = None, - select_subprotocol: Optional[ - Callable[[Sequence[Subprotocol], Sequence[Subprotocol]], Subprotocol] - ] = None, - **kwargs: Any, - ) -> None: - # Backwards compatibility: close_timeout used to be called timeout. - if timeout is None: - timeout = 10 - else: - warnings.warn("rename timeout to close_timeout", DeprecationWarning) - # If both are specified, timeout is ignored. - if close_timeout is None: - close_timeout = timeout - - # Backwards compatibility: create_protocol used to be called klass. - if klass is None: - klass = WebSocketServerProtocol - else: - warnings.warn("rename klass to create_protocol", DeprecationWarning) - # If both are specified, klass is ignored. - if create_protocol is None: - create_protocol = klass - - if loop is None: - loop = asyncio.get_event_loop() - - ws_server = WebSocketServer(loop) - - secure = kwargs.get("ssl") is not None - - if compression == "deflate": - if extensions is None: - extensions = [] - if not any( - ext_factory.name == ServerPerMessageDeflateFactory.name - for ext_factory in extensions - ): - extensions = list(extensions) + [ServerPerMessageDeflateFactory()] - elif compression is not None: - raise ValueError(f"unsupported compression: {compression}") - - factory = functools.partial( - create_protocol, - ws_handler, - ws_server, - host=host, - port=port, - secure=secure, - ping_interval=ping_interval, - ping_timeout=ping_timeout, - close_timeout=close_timeout, - max_size=max_size, - max_queue=max_queue, - read_limit=read_limit, - write_limit=write_limit, - loop=loop, - legacy_recv=legacy_recv, - origins=origins, - extensions=extensions, - subprotocols=subprotocols, - extra_headers=extra_headers, - process_request=process_request, - select_subprotocol=select_subprotocol, - ) - - if path is None: - create_server = functools.partial( - loop.create_server, factory, host, port, **kwargs - ) - else: - # unix_serve(path) must not specify host and port parameters. - assert host is None and port is None - create_server = functools.partial( - loop.create_unix_server, factory, path, **kwargs - ) - - # This is a coroutine function. - self._create_server = create_server - self.ws_server = ws_server - - # async with serve(...) - - async def __aenter__(self) -> WebSocketServer: - return await self - - async def __aexit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> None: - self.ws_server.close() - await self.ws_server.wait_closed() - - # await serve(...) - - def __await__(self) -> Generator[Any, None, WebSocketServer]: - # Create a suitable iterator by calling __await__ on a coroutine. - return self.__await_impl__().__await__() - - async def __await_impl__(self) -> WebSocketServer: - server = await self._create_server() - self.ws_server.wrap(server) - return self.ws_server - - # yield from serve(...) - - __iter__ = __await__ - - -serve = Serve - - -def unix_serve( - ws_handler: Callable[[WebSocketServerProtocol, str], Awaitable[Any]], - path: str, - **kwargs: Any, -) -> Serve: - """ - Similar to :func:`serve`, but for listening on Unix sockets. - - This function calls the event loop's - :meth:`~asyncio.loop.create_unix_server` method. - - It is only available on Unix. - - It's useful for deploying a server behind a reverse proxy such as nginx. - - :param path: file system path to the Unix socket - - """ - return serve(ws_handler, path=path, **kwargs) +__all__ = [ + "serve", + "unix_serve", + "WebSocketServerProtocol", + "WebSocketServer", +] diff --git a/tests/test_client_server.py b/tests/test_asyncio_client_server.py similarity index 98% rename from tests/test_client_server.py rename to tests/test_asyncio_client_server.py index db26d6583..cff76d1f2 100644 --- a/tests/test_client_server.py +++ b/tests/test_asyncio_client_server.py @@ -13,7 +13,8 @@ import urllib.request import warnings -from websockets.client import * +from websockets.asyncio_client import * +from websockets.asyncio_server import * from websockets.datastructures import Headers from websockets.exceptions import ( ConnectionClosed, @@ -31,7 +32,6 @@ from websockets.http import USER_AGENT from websockets.http_legacy import read_response from websockets.protocol import State -from websockets.server import * from websockets.uri import parse_uri from .test_protocol import MS @@ -1072,7 +1072,7 @@ def test_subprotocol_error_two_subprotocols(self, _process_subprotocol): self.run_loop_once() @with_server() - @unittest.mock.patch("websockets.server.read_request") + @unittest.mock.patch("websockets.asyncio_server.read_request") def test_server_receives_malformed_request(self, _read_request): _read_request.side_effect = ValueError("read_request failed") @@ -1080,7 +1080,7 @@ def test_server_receives_malformed_request(self, _read_request): self.start_client() @with_server() - @unittest.mock.patch("websockets.client.read_response") + @unittest.mock.patch("websockets.asyncio_client.read_response") def test_client_receives_malformed_response(self, _read_response): _read_response.side_effect = ValueError("read_response failed") @@ -1089,7 +1089,7 @@ def test_client_receives_malformed_response(self, _read_response): self.run_loop_once() @with_server() - @unittest.mock.patch("websockets.client.build_request") + @unittest.mock.patch("websockets.asyncio_client.build_request") def test_client_sends_invalid_handshake_request(self, _build_request): def wrong_build_request(headers): return "42" @@ -1100,7 +1100,7 @@ def wrong_build_request(headers): self.start_client() @with_server() - @unittest.mock.patch("websockets.server.build_response") + @unittest.mock.patch("websockets.asyncio_server.build_response") def test_server_sends_invalid_handshake_response(self, _build_response): def wrong_build_response(headers, key): return build_response(headers, "42") @@ -1111,7 +1111,7 @@ def wrong_build_response(headers, key): self.start_client() @with_server() - @unittest.mock.patch("websockets.client.read_response") + @unittest.mock.patch("websockets.asyncio_client.read_response") def test_server_does_not_switch_protocols(self, _read_response): async def wrong_read_response(stream): status_code, reason, headers = await read_response(stream) @@ -1124,7 +1124,9 @@ async def wrong_read_response(stream): self.run_loop_once() @with_server() - @unittest.mock.patch("websockets.server.WebSocketServerProtocol.process_request") + @unittest.mock.patch( + "websockets.asyncio_server.WebSocketServerProtocol.process_request" + ) def test_server_error_in_handshake(self, _process_request): _process_request.side_effect = Exception("process_request crashed") @@ -1132,7 +1134,7 @@ def test_server_error_in_handshake(self, _process_request): self.start_client() @with_server() - @unittest.mock.patch("websockets.server.WebSocketServerProtocol.send") + @unittest.mock.patch("websockets.asyncio_server.WebSocketServerProtocol.send") def test_server_handler_crashes(self, send): send.side_effect = ValueError("send failed") @@ -1145,7 +1147,7 @@ def test_server_handler_crashes(self, send): self.assertEqual(self.client.close_code, 1011) @with_server() - @unittest.mock.patch("websockets.server.WebSocketServerProtocol.close") + @unittest.mock.patch("websockets.asyncio_server.WebSocketServerProtocol.close") def test_server_close_crashes(self, close): close.side_effect = ValueError("close failed") @@ -1220,7 +1222,9 @@ def test_invalid_status_error_during_client_connect(self): @unittest.mock.patch( "websockets.server.WebSocketServerProtocol.write_http_response" ) - @unittest.mock.patch("websockets.server.WebSocketServerProtocol.read_http_request") + @unittest.mock.patch( + "websockets.asyncio_server.WebSocketServerProtocol.read_http_request" + ) def test_connection_error_during_opening_handshake( self, _read_http_request, _write_http_response ): @@ -1238,7 +1242,7 @@ def test_connection_error_during_opening_handshake( _write_http_response.assert_not_called() @with_server() - @unittest.mock.patch("websockets.server.WebSocketServerProtocol.close") + @unittest.mock.patch("websockets.asyncio_server.WebSocketServerProtocol.close") def test_connection_error_during_closing_handshake(self, close): close.side_effect = ConnectionError diff --git a/tests/test_auth.py b/tests/test_auth.py index 97a4485a0..c693c9f45 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -6,7 +6,7 @@ from websockets.exceptions import InvalidStatusCode from websockets.headers import build_authorization_basic -from .test_client_server import ClientServerTestsMixin, with_client, with_server +from .test_asyncio_client_server import ClientServerTestsMixin, with_client, with_server from .utils import AsyncioTestCase From 80a8ac8194a9b3591549c6c5bc023f14f1f2c168 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 18 Feb 2020 22:04:00 +0100 Subject: [PATCH 029/104] Implement sans-I/O handshake. --- src/websockets/__init__.py | 2 + src/websockets/asyncio_server.py | 2 +- src/websockets/client.py | 291 ++++++++++++++ src/websockets/connection.py | 88 +++++ src/websockets/events.py | 27 ++ src/websockets/server.py | 426 ++++++++++++++++++++ tests/extensions/utils.py | 76 ++++ tests/test_client.py | 545 ++++++++++++++++++++++++++ tests/test_http11.py | 2 +- tests/test_protocol.py | 10 +- tests/test_server.py | 649 +++++++++++++++++++++++++++++++ tests/utils.py | 4 + 12 files changed, 2115 insertions(+), 7 deletions(-) create mode 100644 src/websockets/connection.py create mode 100644 src/websockets/events.py create mode 100644 tests/extensions/utils.py create mode 100644 tests/test_client.py create mode 100644 tests/test_server.py diff --git a/src/websockets/__init__.py b/src/websockets/__init__.py index 89829235c..c4accaca1 100644 --- a/src/websockets/__init__.py +++ b/src/websockets/__init__.py @@ -15,6 +15,7 @@ "AbortHandshake", "basic_auth_protocol_factory", "BasicAuthWebSocketServerProtocol", + "ClientConnection", "connect", "ConnectionClosed", "ConnectionClosedError", @@ -43,6 +44,7 @@ "RedirectHandshake", "SecurityError", "serve", + "ServerConnection", "Subprotocol", "unix_connect", "unix_serve", diff --git a/src/websockets/asyncio_server.py b/src/websockets/asyncio_server.py index 1eeddf0eb..89ddf6c7d 100644 --- a/src/websockets/asyncio_server.py +++ b/src/websockets/asyncio_server.py @@ -341,7 +341,7 @@ def process_origin( # "The user agent MUST NOT include more than one Origin header field" # per https://tools.ietf.org/html/rfc6454#section-7.3. try: - origin = cast(Origin, headers.get("Origin")) + origin = cast(Optional[Origin], headers.get("Origin")) except MultipleValuesError as exc: raise InvalidHeader("Origin", "more than one Origin header found") from exc if origins is not None: diff --git a/src/websockets/client.py b/src/websockets/client.py index c7d153f13..ec4eb88f5 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -1,8 +1,299 @@ +import collections +import logging +from typing import Generator, List, Optional, Sequence + from .asyncio_client import WebSocketClientProtocol, connect, unix_connect +from .connection import CLIENT, CONNECTING, OPEN, Connection +from .datastructures import Headers, HeadersLike, MultipleValuesError +from .events import Accept, Connect, Event, Reject +from .exceptions import ( + InvalidHandshake, + InvalidHeader, + InvalidHeaderValue, + InvalidStatusCode, + InvalidUpgrade, + NegotiationError, +) +from .extensions.base import ClientExtensionFactory, Extension +from .headers import ( + build_authorization_basic, + build_extension, + build_subprotocol, + parse_connection, + parse_extension, + parse_subprotocol, + parse_upgrade, +) +from .http import USER_AGENT +from .http11 import Request, Response +from .typing import ( + ConnectionOption, + ExtensionHeader, + Origin, + Subprotocol, + UpgradeProtocol, +) +from .uri import parse_uri +from .utils import accept_key, generate_key __all__ = [ "connect", "unix_connect", + "ClientConnection", "WebSocketClientProtocol", ] + +logger = logging.getLogger(__name__) + + +class ClientConnection(Connection): + + side = CLIENT + + def __init__( + self, + uri: str, + origin: Optional[Origin] = None, + extensions: Optional[Sequence[ClientExtensionFactory]] = None, + subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLike] = None, + ): + super().__init__(state=CONNECTING) + self.wsuri = parse_uri(uri) + self.origin = origin + self.available_extensions = extensions + self.available_subprotocols = subprotocols + self.extra_headers = extra_headers + self.key = generate_key() + + def connect(self) -> Connect: + """ + Create a Connect event to send to the server. + + """ + headers = Headers() + + if self.wsuri.port == (443 if self.wsuri.secure else 80): + headers["Host"] = self.wsuri.host + else: + headers["Host"] = f"{self.wsuri.host}:{self.wsuri.port}" + + if self.wsuri.user_info: + headers["Authorization"] = build_authorization_basic(*self.wsuri.user_info) + + if self.origin is not None: + headers["Origin"] = self.origin + + headers["Upgrade"] = "websocket" + headers["Connection"] = "Upgrade" + headers["Sec-WebSocket-Key"] = self.key + headers["Sec-WebSocket-Version"] = "13" + + if self.available_extensions is not None: + extensions_header = build_extension( + [ + (extension_factory.name, extension_factory.get_request_params()) + for extension_factory in self.available_extensions + ] + ) + headers["Sec-WebSocket-Extensions"] = extensions_header + + if self.available_subprotocols is not None: + protocol_header = build_subprotocol(self.available_subprotocols) + headers["Sec-WebSocket-Protocol"] = protocol_header + + if self.extra_headers is not None: + extra_headers = self.extra_headers + if isinstance(extra_headers, Headers): + extra_headers = extra_headers.raw_items() + elif isinstance(extra_headers, collections.abc.Mapping): + extra_headers = extra_headers.items() + for name, value in extra_headers: + headers[name] = value + + headers.setdefault("User-Agent", USER_AGENT) + + request = Request(self.wsuri.resource_name, headers) + return Connect(request) + + def process_response(self, response: Response) -> None: + """ + Check a handshake response received from the server. + + :param response: response + :param key: comes from :func:`build_request` + :raises ~websockets.exceptions.InvalidHandshake: if the handshake response + is invalid + + """ + + if response.status_code != 101: + raise InvalidStatusCode(response.status_code) + + headers = response.headers + + connection: List[ConnectionOption] = sum( + [parse_connection(value) for value in headers.get_all("Connection")], [] + ) + + if not any(value.lower() == "upgrade" for value in connection): + raise InvalidUpgrade( + "Connection", ", ".join(connection) if connection else None + ) + + upgrade: List[UpgradeProtocol] = sum( + [parse_upgrade(value) for value in headers.get_all("Upgrade")], [] + ) + + # For compatibility with non-strict implementations, ignore case when + # checking the Upgrade header. It's supposed to be 'WebSocket'. + if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): + raise InvalidUpgrade("Upgrade", ", ".join(upgrade) if upgrade else None) + + try: + s_w_accept = headers["Sec-WebSocket-Accept"] + except KeyError: + raise InvalidHeader("Sec-WebSocket-Accept") + except MultipleValuesError: + raise InvalidHeader( + "Sec-WebSocket-Accept", + "more than one Sec-WebSocket-Accept header found", + ) + + if s_w_accept != accept_key(self.key): + raise InvalidHeaderValue("Sec-WebSocket-Accept", s_w_accept) + + self.extensions = self.process_extensions(headers) + + self.subprotocol = self.process_subprotocol(headers) + + def process_extensions(self, headers: Headers) -> List[Extension]: + """ + Handle the Sec-WebSocket-Extensions HTTP response header. + + Check that each extension is supported, as well as its parameters. + + Return the list of accepted extensions. + + Raise :exc:`~websockets.exceptions.InvalidHandshake` to abort the + connection. + + :rfc:`6455` leaves the rules up to the specification of each + extension. + + To provide this level of flexibility, for each extension accepted by + the server, we check for a match with each extension available in the + client configuration. If no match is found, an exception is raised. + + If several variants of the same extension are accepted by the server, + it may be configured severel times, which won't make sense in general. + Extensions must implement their own requirements. For this purpose, + the list of previously accepted extensions is provided. + + Other requirements, for example related to mandatory extensions or the + order of extensions, may be implemented by overriding this method. + + """ + accepted_extensions: List[Extension] = [] + + extensions = headers.get_all("Sec-WebSocket-Extensions") + + if extensions: + + if self.available_extensions is None: + raise InvalidHandshake("no extensions supported") + + parsed_extensions: List[ExtensionHeader] = sum( + [parse_extension(header_value) for header_value in extensions], [] + ) + + for name, response_params in parsed_extensions: + + for extension_factory in self.available_extensions: + + # Skip non-matching extensions based on their name. + if extension_factory.name != name: + continue + + # Skip non-matching extensions based on their params. + try: + extension = extension_factory.process_response_params( + response_params, accepted_extensions + ) + except NegotiationError: + continue + + # Add matching extension to the final list. + accepted_extensions.append(extension) + + # Break out of the loop once we have a match. + break + + # If we didn't break from the loop, no extension in our list + # matched what the server sent. Fail the connection. + else: + raise NegotiationError( + f"Unsupported extension: " + f"name = {name}, params = {response_params}" + ) + + return accepted_extensions + + def process_subprotocol(self, headers: Headers) -> Optional[Subprotocol]: + """ + Handle the Sec-WebSocket-Protocol HTTP response header. + + Check that it contains exactly one supported subprotocol. + + Return the selected subprotocol. + + """ + subprotocol: Optional[Subprotocol] = None + + subprotocols = headers.get_all("Sec-WebSocket-Protocol") + + if subprotocols: + + if self.available_subprotocols is None: + raise InvalidHandshake("no subprotocols supported") + + parsed_subprotocols: Sequence[Subprotocol] = sum( + [parse_subprotocol(header_value) for header_value in subprotocols], [] + ) + + if len(parsed_subprotocols) > 1: + subprotocols_display = ", ".join(parsed_subprotocols) + raise InvalidHandshake(f"multiple subprotocols: {subprotocols_display}") + + subprotocol = parsed_subprotocols[0] + + if subprotocol not in self.available_subprotocols: + raise NegotiationError(f"unsupported subprotocol: {subprotocol}") + + return subprotocol + + def send_in_connecting_state(self, event: Event) -> bytes: + assert isinstance(event, Connect) + + request = event.request + + logger.debug("%s > GET %s HTTP/1.1", self.side, request.path) + logger.debug("%s > %r", self.side, request.headers) + + return request.serialize() + + def parse(self) -> Generator[None, None, None]: + response = yield from Response.parse( + self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof, + ) + assert self.state == CONNECTING + try: + self.process_response(response) + except InvalidHandshake as exc: + self.events.append(Reject(response, exc)) + return + else: + self.events.append(Accept(response)) + self.state = OPEN + yield from super().parse() diff --git a/src/websockets/connection.py b/src/websockets/connection.py new file mode 100644 index 000000000..5789b6ea1 --- /dev/null +++ b/src/websockets/connection.py @@ -0,0 +1,88 @@ +import enum +from typing import Generator, Iterable, List, Tuple + +from .events import Event +from .exceptions import InvalidState +from .streams import StreamReader + + +__all__ = ["Connection"] + + +# A WebSocket connection is either a server or a client. + + +class Side(enum.IntEnum): + SERVER, CLIENT = range(2) + + +SERVER = Side.SERVER +CLIENT = Side.CLIENT + + +# A WebSocket connection goes through the following four states, in order: + + +class State(enum.IntEnum): + CONNECTING, OPEN, CLOSING, CLOSED = range(4) + + +CONNECTING = State.CONNECTING +OPEN = State.OPEN +CLOSING = State.CLOSING +CLOSED = State.CLOSED + + +class Connection: + + side: Side + + def __init__(self, state: State = OPEN) -> None: + self.state = state + self.reader = StreamReader() + self.events: List[Event] = [] + self.parser = self.parse() + next(self.parser) # start coroutine + + # Public APIs for receiving data and producing events + + def receive_data(self, data: bytes) -> Tuple[Iterable[Event], bytes]: + self.reader.feed_data(data) + return self.receive() + + def receive_eof(self) -> Tuple[Iterable[Event], bytes]: + self.reader.feed_eof() + return self.receive() + + # Public APIs for receiving events and producing data + + def send(self, event: Event) -> bytes: + """ + Send an event to the remote endpoint. + + """ + if self.state == OPEN: + raise NotImplementedError # not implemented yet + elif self.state == CONNECTING: + return self.send_in_connecting_state(event) + else: + raise InvalidState( + f"Cannot write to a WebSocket in the {self.state.name} state" + ) + + # Private APIs + + def send_in_connecting_state(self, event: Event) -> bytes: + raise NotImplementedError + + def receive(self) -> Tuple[List[Event], bytes]: + # Run parser until more data is needed or EOF + try: + next(self.parser) + except StopIteration: + pass + events, self.events = self.events, [] + return events, b"" + + def parse(self) -> Generator[None, None, None]: + yield # not implemented yet diff --git a/src/websockets/events.py b/src/websockets/events.py new file mode 100644 index 000000000..196de9421 --- /dev/null +++ b/src/websockets/events.py @@ -0,0 +1,27 @@ +from typing import NamedTuple, Optional, Union + +from .http11 import Request, Response + + +__all__ = [ + "Accept", + "Connect", + "Event", + "Reject", +] + + +class Connect(NamedTuple): + request: Request + + +class Accept(NamedTuple): + response: Response + + +class Reject(NamedTuple): + response: Response + exception: Optional[Exception] + + +Event = Union[Connect, Accept, Reject] diff --git a/src/websockets/server.py b/src/websockets/server.py index ec94a2fbf..f668ff5e7 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -1,9 +1,435 @@ +import base64 +import binascii +import collections +import email.utils +import http +import logging +from typing import Callable, Generator, List, Optional, Sequence, Tuple, Union, cast + from .asyncio_server import WebSocketServer, WebSocketServerProtocol, serve, unix_serve +from .connection import CONNECTING, OPEN, SERVER, Connection +from .datastructures import Headers, HeadersLike, MultipleValuesError +from .events import Accept, Connect, Event, Reject +from .exceptions import ( + InvalidHandshake, + InvalidHeader, + InvalidHeaderValue, + InvalidOrigin, + InvalidUpgrade, + NegotiationError, +) +from .extensions.base import Extension, ServerExtensionFactory +from .headers import ( + build_extension, + parse_connection, + parse_extension, + parse_subprotocol, + parse_upgrade, +) +from .http import USER_AGENT +from .http11 import Request, Response +from .typing import ( + ConnectionOption, + ExtensionHeader, + Origin, + Subprotocol, + UpgradeProtocol, +) +from .utils import accept_key __all__ = [ "serve", "unix_serve", + "ServerConnection", "WebSocketServerProtocol", "WebSocketServer", ] + +logger = logging.getLogger(__name__) + + +HeadersLikeOrCallable = Union[HeadersLike, Callable[[str, Headers], HeadersLike]] + + +class ServerConnection(Connection): + + side = SERVER + + def __init__( + self, + origins: Optional[Sequence[Optional[Origin]]] = None, + extensions: Optional[Sequence[ServerExtensionFactory]] = None, + subprotocols: Optional[Sequence[Subprotocol]] = None, + extra_headers: Optional[HeadersLikeOrCallable] = None, + ): + super().__init__(state=CONNECTING) + self.origins = origins + self.available_extensions = extensions + self.available_subprotocols = subprotocols + self.extra_headers = extra_headers + + def accept(self, connect: Connect) -> Union[Accept, Reject]: + """ + Create an ``Accept`` or ``Reject`` event to send to the client. + + If the connection cannot be established, this method returns a + :class:`~websockets.events.Reject` event, which may be unexpected. + + """ + request = connect.request + try: + key, extensions_header, protocol_header = self.process_request(request) + except InvalidOrigin as exc: + logger.debug("Invalid origin", exc_info=True) + return self.reject( + http.HTTPStatus.FORBIDDEN, + f"Failed to open a WebSocket connection: {exc}.\n", + exception=exc, + ) + except InvalidUpgrade as exc: + logger.debug("Invalid upgrade", exc_info=True) + return self.reject( + http.HTTPStatus.UPGRADE_REQUIRED, + ( + f"Failed to open a WebSocket connection: {exc}.\n" + f"\n" + f"You cannot access a WebSocket server directly " + f"with a browser. You need a WebSocket client.\n" + ), + headers=Headers([("Upgrade", "websocket")]), + exception=exc, + ) + except InvalidHandshake as exc: + logger.debug("Invalid handshake", exc_info=True) + return self.reject( + http.HTTPStatus.BAD_REQUEST, + f"Failed to open a WebSocket connection: {exc}.\n", + exception=exc, + ) + except Exception as exc: + logger.warning("Error in opening handshake", exc_info=True) + return self.reject( + http.HTTPStatus.INTERNAL_SERVER_ERROR, + ( + "Failed to open a WebSocket connection.\n" + "See server log for more information.\n" + ), + exception=exc, + ) + + headers = Headers() + + headers["Upgrade"] = "websocket" + headers["Connection"] = "Upgrade" + headers["Sec-WebSocket-Accept"] = accept_key(key) + + if extensions_header is not None: + headers["Sec-WebSocket-Extensions"] = extensions_header + + if protocol_header is not None: + headers["Sec-WebSocket-Protocol"] = protocol_header + + extra_headers: Optional[HeadersLike] + if callable(self.extra_headers): + extra_headers = self.extra_headers(request.path, request.headers) + else: + extra_headers = self.extra_headers + if extra_headers is not None: + if isinstance(extra_headers, Headers): + extra_headers = extra_headers.raw_items() + elif isinstance(extra_headers, collections.abc.Mapping): + extra_headers = extra_headers.items() + for name, value in extra_headers: + headers[name] = value + + headers.setdefault("Date", email.utils.formatdate(usegmt=True)) + headers.setdefault("Server", USER_AGENT) + + response = Response(101, "Switching Protocols", headers) + return Accept(response) + + def process_request( + self, request: Request + ) -> Tuple[str, Optional[str], Optional[str]]: + """ + Check a handshake request received from the client. + + This function doesn't verify that the request is an HTTP/1.1 or higher GET + request and doesn't perform ``Host`` and ``Origin`` checks. These controls + are usually performed earlier in the HTTP request handling code. They're + the responsibility of the caller. + + :param request: request + :returns: ``key`` which must be passed to :func:`build_response` + :raises ~websockets.exceptions.InvalidHandshake: if the handshake request + is invalid; then the server must return 400 Bad Request error + + """ + headers = request.headers + + connection: List[ConnectionOption] = sum( + [parse_connection(value) for value in headers.get_all("Connection")], [] + ) + + if not any(value.lower() == "upgrade" for value in connection): + raise InvalidUpgrade( + "Connection", ", ".join(connection) if connection else None + ) + + upgrade: List[UpgradeProtocol] = sum( + [parse_upgrade(value) for value in headers.get_all("Upgrade")], [] + ) + + # For compatibility with non-strict implementations, ignore case when + # checking the Upgrade header. The RFC always uses "websocket", except + # in section 11.2. (IANA registration) where it uses "WebSocket". + if not (len(upgrade) == 1 and upgrade[0].lower() == "websocket"): + raise InvalidUpgrade("Upgrade", ", ".join(upgrade) if upgrade else None) + + try: + key = headers["Sec-WebSocket-Key"] + except KeyError: + raise InvalidHeader("Sec-WebSocket-Key") + except MultipleValuesError: + raise InvalidHeader( + "Sec-WebSocket-Key", "more than one Sec-WebSocket-Key header found" + ) + + try: + raw_key = base64.b64decode(key.encode(), validate=True) + except binascii.Error: + raise InvalidHeaderValue("Sec-WebSocket-Key", key) + if len(raw_key) != 16: + raise InvalidHeaderValue("Sec-WebSocket-Key", key) + + try: + version = headers["Sec-WebSocket-Version"] + except KeyError: + raise InvalidHeader("Sec-WebSocket-Version") + except MultipleValuesError: + raise InvalidHeader( + "Sec-WebSocket-Version", + "more than one Sec-WebSocket-Version header found", + ) + + if version != "13": + raise InvalidHeaderValue("Sec-WebSocket-Version", version) + + self.origin = self.process_origin(headers) + + extensions_header, self.extensions = self.process_extensions(headers) + + protocol_header = self.subprotocol = self.process_subprotocol(headers) + + return key, extensions_header, protocol_header + + def process_origin(self, headers: Headers) -> Optional[Origin]: + """ + Handle the Origin HTTP request header. + + :param headers: request headers + :raises ~websockets.exceptions.InvalidOrigin: if the origin isn't + acceptable + + """ + # "The user agent MUST NOT include more than one Origin header field" + # per https://tools.ietf.org/html/rfc6454#section-7.3. + try: + origin = cast(Optional[Origin], headers.get("Origin")) + except MultipleValuesError as exc: + raise InvalidHeader("Origin", "more than one Origin header found") from exc + if self.origins is not None: + if origin not in self.origins: + raise InvalidOrigin(origin) + return origin + + def process_extensions( + self, headers: Headers, + ) -> Tuple[Optional[str], List[Extension]]: + """ + Handle the Sec-WebSocket-Extensions HTTP request header. + + Accept or reject each extension proposed in the client request. + Negotiate parameters for accepted extensions. + + Return the Sec-WebSocket-Extensions HTTP response header and the list + of accepted extensions. + + :rfc:`6455` leaves the rules up to the specification of each + :extension. + + To provide this level of flexibility, for each extension proposed by + the client, we check for a match with each extension available in the + server configuration. If no match is found, the extension is ignored. + + If several variants of the same extension are proposed by the client, + it may be accepted several times, which won't make sense in general. + Extensions must implement their own requirements. For this purpose, + the list of previously accepted extensions is provided. + + This process doesn't allow the server to reorder extensions. It can + only select a subset of the extensions proposed by the client. + + Other requirements, for example related to mandatory extensions or the + order of extensions, may be implemented by overriding this method. + + :param headers: request headers + :raises ~websockets.exceptions.InvalidHandshake: to abort the + handshake with an HTTP 400 error code + + """ + response_header_value: Optional[str] = None + + extension_headers: List[ExtensionHeader] = [] + accepted_extensions: List[Extension] = [] + + header_values = headers.get_all("Sec-WebSocket-Extensions") + + if header_values and self.available_extensions: + + parsed_header_values: List[ExtensionHeader] = sum( + [parse_extension(header_value) for header_value in header_values], [] + ) + + for name, request_params in parsed_header_values: + + for ext_factory in self.available_extensions: + + # Skip non-matching extensions based on their name. + if ext_factory.name != name: + continue + + # Skip non-matching extensions based on their params. + try: + response_params, extension = ext_factory.process_request_params( + request_params, accepted_extensions + ) + except NegotiationError: + continue + + # Add matching extension to the final list. + extension_headers.append((name, response_params)) + accepted_extensions.append(extension) + + # Break out of the loop once we have a match. + break + + # If we didn't break from the loop, no extension in our list + # matched what the client sent. The extension is declined. + + # Serialize extension header. + if extension_headers: + response_header_value = build_extension(extension_headers) + + return response_header_value, accepted_extensions + + def process_subprotocol(self, headers: Headers) -> Optional[Subprotocol]: + """ + Handle the Sec-WebSocket-Protocol HTTP request header. + + Return Sec-WebSocket-Protocol HTTP response header, which is the same + as the selected subprotocol. + + :param headers: request headers + :raises ~websockets.exceptions.InvalidHandshake: to abort the + handshake with an HTTP 400 error code + + """ + subprotocol: Optional[Subprotocol] = None + + header_values = headers.get_all("Sec-WebSocket-Protocol") + + if header_values and self.available_subprotocols: + + parsed_header_values: List[Subprotocol] = sum( + [parse_subprotocol(header_value) for header_value in header_values], [] + ) + + subprotocol = self.select_subprotocol( + parsed_header_values, self.available_subprotocols + ) + + return subprotocol + + def select_subprotocol( + self, + client_subprotocols: Sequence[Subprotocol], + server_subprotocols: Sequence[Subprotocol], + ) -> Optional[Subprotocol]: + """ + Pick a subprotocol among those offered by the client. + + If several subprotocols are supported by the client and the server, + the default implementation selects the preferred subprotocols by + giving equal value to the priorities of the client and the server. + + If no common subprotocol is supported by the client and the server, it + proceeds without a subprotocol. + + This is unlikely to be the most useful implementation in practice, as + many servers providing a subprotocol will require that the client uses + that subprotocol. + + :param client_subprotocols: list of subprotocols offered by the client + :param server_subprotocols: list of subprotocols available on the server + + """ + subprotocols = set(client_subprotocols) & set(server_subprotocols) + if not subprotocols: + return None + priority = lambda p: ( + client_subprotocols.index(p) + server_subprotocols.index(p) + ) + return sorted(subprotocols, key=priority)[0] + + def reject( + self, + status: http.HTTPStatus, + text: str, + headers: Optional[Headers] = None, + exception: Optional[Exception] = None, + ) -> Reject: + """ + Create a ``Reject`` event to send to the client. + + A short plain text response is the best fallback when failing to + establish a WebSocket connection. + + """ + body = text.encode() + if headers is None: + headers = Headers() + headers.setdefault("Date", email.utils.formatdate(usegmt=True)) + headers.setdefault("Server", USER_AGENT) + headers.setdefault("Content-Length", str(len(body))) + headers.setdefault("Content-Type", "text/plain; charset=utf-8") + headers.setdefault("Connection", "close") + response = Response(status.value, status.phrase, headers, body) + return Reject(response, exception) + + def send_in_connecting_state(self, event: Event) -> bytes: + assert isinstance(event, (Accept, Reject)) + + if isinstance(event, Accept): + self.state = OPEN + + response = event.response + + logger.debug( + "%s > HTTP/1.1 %d %s", + self.side, + response.status_code, + response.reason_phrase, + ) + logger.debug("%s > %r", self.side, response.headers) + if response.body is not None: + logger.debug("%s > body (%d bytes)", self.side, len(response.body)) + + return response.serialize() + + def parse(self) -> Generator[None, None, None]: + request = yield from Request.parse(self.reader.read_line) + assert self.state == CONNECTING + self.events.append(Connect(request)) + yield from super().parse() diff --git a/tests/extensions/utils.py b/tests/extensions/utils.py new file mode 100644 index 000000000..81990bb07 --- /dev/null +++ b/tests/extensions/utils.py @@ -0,0 +1,76 @@ +from websockets.exceptions import NegotiationError + + +class OpExtension: + name = "x-op" + + def __init__(self, op=None): + self.op = op + + def decode(self, frame, *, max_size=None): + return frame # pragma: no cover + + def encode(self, frame): + return frame # pragma: no cover + + def __eq__(self, other): + return isinstance(other, OpExtension) and self.op == other.op + + +class ClientOpExtensionFactory: + name = "x-op" + + def __init__(self, op=None): + self.op = op + + def get_request_params(self): + return [("op", self.op)] + + def process_response_params(self, params, accepted_extensions): + if params != [("op", self.op)]: + raise NegotiationError() + return OpExtension(self.op) + + +class ServerOpExtensionFactory: + name = "x-op" + + def __init__(self, op=None): + self.op = op + + def process_request_params(self, params, accepted_extensions): + if params != [("op", self.op)]: + raise NegotiationError() + return [("op", self.op)], OpExtension(self.op) + + +class Rsv2Extension: + name = "x-rsv2" + + def decode(self, frame, *, max_size=None): + assert frame.rsv2 + return frame._replace(rsv2=False) + + def encode(self, frame): + assert not frame.rsv2 + return frame._replace(rsv2=True) + + def __eq__(self, other): + return isinstance(other, Rsv2Extension) + + +class ClientRsv2ExtensionFactory: + name = "x-rsv2" + + def get_request_params(self): + return [] + + def process_response_params(self, params, accepted_extensions): + return Rsv2Extension() + + +class ServerRsv2ExtensionFactory: + name = "x-rsv2" + + def process_request_params(self, params, accepted_extensions): + return [], Rsv2Extension() diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 000000000..1cf27349d --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,545 @@ +import unittest +import unittest.mock + +from websockets.client import * +from websockets.connection import CONNECTING, OPEN +from websockets.datastructures import Headers +from websockets.events import Accept, Connect, Reject +from websockets.exceptions import InvalidHandshake, InvalidHeader +from websockets.http import USER_AGENT +from websockets.http11 import Request, Response +from websockets.utils import accept_key + +from .extensions.utils import ( + ClientOpExtensionFactory, + ClientRsv2ExtensionFactory, + OpExtension, + Rsv2Extension, +) +from .test_utils import ACCEPT, KEY +from .utils import DATE + + +class ConnectTests(unittest.TestCase): + def test_send_connect(self): + with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): + client = ClientConnection("wss://example.com/test") + connect = client.connect() + self.assertIsInstance(connect, Connect) + bytes_to_send = client.send(connect) + self.assertEqual( + bytes_to_send, + ( + f"GET /test HTTP/1.1\r\n" + f"Host: example.com\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {KEY}\r\n" + f"Sec-WebSocket-Version: 13\r\n" + f"User-Agent: {USER_AGENT}\r\n" + f"\r\n" + ).encode(), + ) + + def test_connect_request(self): + with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): + client = ClientConnection("wss://example.com/test") + connect = client.connect() + self.assertIsInstance(connect.request, Request) + self.assertEqual(connect.request.path, "/test") + self.assertEqual( + connect.request.headers, + Headers( + { + "Host": "example.com", + "Upgrade": "websocket", + "Connection": "Upgrade", + "Sec-WebSocket-Key": KEY, + "Sec-WebSocket-Version": "13", + "User-Agent": USER_AGENT, + } + ), + ) + + def test_path(self): + client = ClientConnection("wss://example.com/endpoint?test=1") + request = client.connect().request + + self.assertEqual(request.path, "/endpoint?test=1") + + def test_port(self): + for uri, host in [ + ("ws://example.com/", "example.com"), + ("ws://example.com:80/", "example.com"), + ("ws://example.com:8080/", "example.com:8080"), + ("wss://example.com/", "example.com"), + ("wss://example.com:443/", "example.com"), + ("wss://example.com:8443/", "example.com:8443"), + ]: + with self.subTest(uri=uri): + client = ClientConnection(uri) + request = client.connect().request + + self.assertEqual(request.headers["Host"], host) + + def test_user_info(self): + client = ClientConnection("wss://hello:iloveyou@example.com/") + request = client.connect().request + + self.assertEqual(request.headers["Authorization"], "Basic aGVsbG86aWxvdmV5b3U=") + + def test_origin(self): + client = ClientConnection("wss://example.com/", origin="https://example.com") + request = client.connect().request + + self.assertEqual(request.headers["Origin"], "https://example.com") + + def test_extensions(self): + client = ClientConnection( + "wss://example.com/", extensions=[ClientOpExtensionFactory()] + ) + request = client.connect().request + + self.assertEqual(request.headers["Sec-WebSocket-Extensions"], "x-op; op") + + def test_subprotocols(self): + client = ClientConnection("wss://example.com/", subprotocols=["chat"]) + request = client.connect().request + + self.assertEqual(request.headers["Sec-WebSocket-Protocol"], "chat") + + def test_extra_headers(self): + for extra_headers in [ + Headers({"X-Spam": "Eggs"}), + {"X-Spam": "Eggs"}, + [("X-Spam", "Eggs")], + ]: + with self.subTest(extra_headers=extra_headers): + client = ClientConnection( + "wss://example.com/", extra_headers=extra_headers + ) + request = client.connect().request + + self.assertEqual(request.headers["X-Spam"], "Eggs") + + def test_extra_headers_overrides_user_agent(self): + client = ClientConnection( + "wss://example.com/", extra_headers={"User-Agent": "Other"} + ) + request = client.connect().request + + self.assertEqual(request.headers["User-Agent"], "Other") + + +class AcceptRejectTests(unittest.TestCase): + def test_receive_accept(self): + with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): + client = ClientConnection("ws://example.com/test") + client.connect() + [accept], bytes_to_send = client.receive_data( + ( + f"HTTP/1.1 101 Switching Protocols\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Accept: {ACCEPT}\r\n" + f"Date: {DATE}\r\n" + f"Server: {USER_AGENT}\r\n" + f"\r\n" + ).encode(), + ) + self.assertIsInstance(accept, Accept) + self.assertEqual(bytes_to_send, b"") + self.assertEqual(client.state, OPEN) + + def test_receive_reject(self): + with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): + client = ClientConnection("ws://example.com/test") + client.connect() + [reject], bytes_to_send = client.receive_data( + ( + f"HTTP/1.1 404 Not Found\r\n" + f"Date: {DATE}\r\n" + f"Server: {USER_AGENT}\r\n" + f"Content-Length: 13\r\n" + f"Content-Type: text/plain; charset=utf-8\r\n" + f"Connection: close\r\n" + f"\r\n" + f"Sorry folks.\n" + ).encode(), + ) + self.assertIsInstance(reject, Reject) + self.assertEqual(bytes_to_send, b"") + self.assertEqual(client.state, CONNECTING) + + def test_accept_response(self): + with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): + client = ClientConnection("ws://example.com/test") + client.connect() + [accept], _bytes_to_send = client.receive_data( + ( + f"HTTP/1.1 101 Switching Protocols\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Accept: {ACCEPT}\r\n" + f"Date: {DATE}\r\n" + f"Server: {USER_AGENT}\r\n" + f"\r\n" + ).encode(), + ) + self.assertEqual(accept.response.status_code, 101) + self.assertEqual(accept.response.reason_phrase, "Switching Protocols") + self.assertEqual( + accept.response.headers, + Headers( + { + "Upgrade": "websocket", + "Connection": "Upgrade", + "Sec-WebSocket-Accept": ACCEPT, + "Date": DATE, + "Server": USER_AGENT, + } + ), + ) + self.assertIsNone(accept.response.body) + + def test_reject_response(self): + with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): + client = ClientConnection("ws://example.com/test") + client.connect() + [reject], _bytes_to_send = client.receive_data( + ( + f"HTTP/1.1 404 Not Found\r\n" + f"Date: {DATE}\r\n" + f"Server: {USER_AGENT}\r\n" + f"Content-Length: 13\r\n" + f"Content-Type: text/plain; charset=utf-8\r\n" + f"Connection: close\r\n" + f"\r\n" + f"Sorry folks.\n" + ).encode(), + ) + self.assertEqual(reject.response.status_code, 404) + self.assertEqual(reject.response.reason_phrase, "Not Found") + self.assertEqual( + reject.response.headers, + Headers( + { + "Date": DATE, + "Server": USER_AGENT, + "Content-Length": "13", + "Content-Type": "text/plain; charset=utf-8", + "Connection": "close", + } + ), + ) + self.assertEqual(reject.response.body, b"Sorry folks.\n") + + def make_accept_response(self, client): + request = client.connect().request + return Response( + status_code=101, + reason_phrase="Switching Protocols", + headers=Headers( + { + "Upgrade": "websocket", + "Connection": "Upgrade", + "Sec-WebSocket-Accept": accept_key( + request.headers["Sec-WebSocket-Key"] + ), + } + ), + ) + + def test_basic(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + + def test_missing_connection(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + del response.headers["Connection"] + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Connection header") + + def test_invalid_connection(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + del response.headers["Connection"] + response.headers["Connection"] = "close" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "invalid Connection header: close") + + def test_missing_upgrade(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + del response.headers["Upgrade"] + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Upgrade header") + + def test_invalid_upgrade(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + del response.headers["Upgrade"] + response.headers["Upgrade"] = "h2c" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "invalid Upgrade header: h2c") + + def test_missing_accept(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + del response.headers["Sec-WebSocket-Accept"] + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Sec-WebSocket-Accept header") + + def test_multiple_accept(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Accept"] = ACCEPT + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), + "invalid Sec-WebSocket-Accept header: " + "more than one Sec-WebSocket-Accept header found", + ) + + def test_invalid_accept(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + del response.headers["Sec-WebSocket-Accept"] + response.headers["Sec-WebSocket-Accept"] = ACCEPT + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), f"invalid Sec-WebSocket-Accept header: {ACCEPT}" + ) + + def test_no_extensions(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.extensions, []) + + def test_no_extension(self): + client = ClientConnection( + "wss://example.com/", extensions=[ClientOpExtensionFactory()] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-op; op" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.extensions, [OpExtension()]) + + def test_extension(self): + client = ClientConnection( + "wss://example.com/", extensions=[ClientRsv2ExtensionFactory()] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.extensions, [Rsv2Extension()]) + + def test_unexpected_extension(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-op; op" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHandshake) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "no extensions supported") + + def test_unsupported_extension(self): + client = ClientConnection( + "wss://example.com/", extensions=[ClientRsv2ExtensionFactory()] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-op; op" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHandshake) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), + "Unsupported extension: name = x-op, params = [('op', None)]", + ) + + def test_supported_extension_parameters(self): + client = ClientConnection( + "wss://example.com/", extensions=[ClientOpExtensionFactory("this")] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-op; op=this" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.extensions, [OpExtension("this")]) + + def test_unsupported_extension_parameters(self): + client = ClientConnection( + "wss://example.com/", extensions=[ClientOpExtensionFactory("this")] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHandshake) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), + "Unsupported extension: name = x-op, params = [('op', 'that')]", + ) + + def test_multiple_supported_extension_parameters(self): + client = ClientConnection( + "wss://example.com/", + extensions=[ + ClientOpExtensionFactory("this"), + ClientOpExtensionFactory("that"), + ], + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.extensions, [OpExtension("that")]) + + def test_multiple_extensions(self): + client = ClientConnection( + "wss://example.com/", + extensions=[ClientOpExtensionFactory(), ClientRsv2ExtensionFactory()], + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-op; op" + response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.extensions, [OpExtension(), Rsv2Extension()]) + + def test_multiple_extensions_order(self): + client = ClientConnection( + "wss://example.com/", + extensions=[ClientOpExtensionFactory(), ClientRsv2ExtensionFactory()], + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" + response.headers["Sec-WebSocket-Extensions"] = "x-op; op" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.extensions, [Rsv2Extension(), OpExtension()]) + + def test_no_subprotocols(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertIsNone(client.subprotocol) + + def test_no_subprotocol(self): + client = ClientConnection("wss://example.com/", subprotocols=["chat"]) + response = self.make_accept_response(client) + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertIsNone(client.subprotocol) + + def test_subprotocol(self): + client = ClientConnection("wss://example.com/", subprotocols=["chat"]) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Protocol"] = "chat" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.subprotocol, "chat") + + def test_unexpected_subprotocol(self): + client = ClientConnection("wss://example.com/") + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Protocol"] = "chat" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHandshake) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "no subprotocols supported") + + def test_multiple_subprotocols(self): + client = ClientConnection( + "wss://example.com/", subprotocols=["superchat", "chat"] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Protocol"] = "superchat" + response.headers["Sec-WebSocket-Protocol"] = "chat" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHandshake) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), "multiple subprotocols: superchat, chat" + ) + + def test_supported_subprotocol(self): + client = ClientConnection( + "wss://example.com/", subprotocols=["superchat", "chat"] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Protocol"] = "chat" + [accept], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(accept, Accept) + self.assertEqual(client.subprotocol, "chat") + + def test_unsupported_subprotocol(self): + client = ClientConnection( + "wss://example.com/", subprotocols=["superchat", "chat"] + ) + response = self.make_accept_response(client) + response.headers["Sec-WebSocket-Protocol"] = "otherchat" + [reject], _bytes_to_send = client.receive_data(response.serialize()) + + self.assertIsInstance(reject, Reject) + with self.assertRaises(InvalidHandshake) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "unsupported subprotocol: otherchat") diff --git a/tests/test_http11.py b/tests/test_http11.py index bca874aee..4574cf97e 100644 --- a/tests/test_http11.py +++ b/tests/test_http11.py @@ -101,7 +101,7 @@ def setUp(self): def parse(self): return Response.parse( - self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof + self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof, ) def test_parse(self): diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 91fb02a50..3054600e1 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -553,13 +553,13 @@ def test_recv_when_transfer_data_cancelled(self): def test_recv_prevents_concurrent_calls(self): recv = self.loop.create_task(self.protocol.recv()) - with self.assertRaisesRegex( - RuntimeError, + with self.assertRaises(RuntimeError) as raised: + self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual( + str(raised.exception), "cannot call recv while another coroutine " "is already waiting for the next message", - ): - self.loop.run_until_complete(self.protocol.recv()) - + ) recv.cancel() # Test the send coroutine. diff --git a/tests/test_server.py b/tests/test_server.py new file mode 100644 index 000000000..1d094a86d --- /dev/null +++ b/tests/test_server.py @@ -0,0 +1,649 @@ +import http +import unittest +import unittest.mock + +from websockets.connection import CONNECTING, OPEN +from websockets.datastructures import Headers +from websockets.events import Accept, Connect, Reject +from websockets.exceptions import InvalidHeader, InvalidOrigin, InvalidUpgrade +from websockets.http import USER_AGENT +from websockets.http11 import Request, Response +from websockets.server import * + +from .extensions.utils import ( + OpExtension, + Rsv2Extension, + ServerOpExtensionFactory, + ServerRsv2ExtensionFactory, +) +from .test_utils import ACCEPT, KEY +from .utils import DATE + + +class ConnectTests(unittest.TestCase): + def test_receive_connect(self): + server = ServerConnection() + [connect], bytes_to_send = server.receive_data( + ( + f"GET /test HTTP/1.1\r\n" + f"Host: example.com\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {KEY}\r\n" + f"Sec-WebSocket-Version: 13\r\n" + f"User-Agent: {USER_AGENT}\r\n" + f"\r\n" + ).encode(), + ) + self.assertIsInstance(connect, Connect) + self.assertEqual(bytes_to_send, b"") + + def test_connect_request(self): + server = ServerConnection() + [connect], bytes_to_send = server.receive_data( + ( + f"GET /test HTTP/1.1\r\n" + f"Host: example.com\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {KEY}\r\n" + f"Sec-WebSocket-Version: 13\r\n" + f"User-Agent: {USER_AGENT}\r\n" + f"\r\n" + ).encode(), + ) + self.assertEqual(connect.request.path, "/test") + self.assertEqual( + connect.request.headers, + Headers( + { + "Host": "example.com", + "Upgrade": "websocket", + "Connection": "Upgrade", + "Sec-WebSocket-Key": KEY, + "Sec-WebSocket-Version": "13", + "User-Agent": USER_AGENT, + } + ), + ) + + +class AcceptRejectTests(unittest.TestCase): + def make_connect_request(self): + return Request( + path="/test", + headers=Headers( + { + "Host": "example.com", + "Upgrade": "websocket", + "Connection": "Upgrade", + "Sec-WebSocket-Key": KEY, + "Sec-WebSocket-Version": "13", + "User-Agent": USER_AGENT, + } + ), + ) + + def test_send_accept(self): + server = ServerConnection() + with unittest.mock.patch("email.utils.formatdate", return_value=DATE): + accept = server.accept(Connect(self.make_connect_request())) + self.assertIsInstance(accept, Accept) + bytes_to_send = server.send(accept) + self.assertEqual( + bytes_to_send, + ( + f"HTTP/1.1 101 Switching Protocols\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Accept: {ACCEPT}\r\n" + f"Date: {DATE}\r\n" + f"Server: {USER_AGENT}\r\n" + f"\r\n" + ).encode(), + ) + self.assertEqual(server.state, OPEN) + + def test_send_reject(self): + server = ServerConnection() + with unittest.mock.patch("email.utils.formatdate", return_value=DATE): + reject = server.reject(http.HTTPStatus.NOT_FOUND, "Sorry folks.\n") + self.assertIsInstance(reject, Reject) + bytes_to_send = server.send(reject) + self.assertEqual( + bytes_to_send, + ( + f"HTTP/1.1 404 Not Found\r\n" + f"Date: {DATE}\r\n" + f"Server: {USER_AGENT}\r\n" + f"Content-Length: 13\r\n" + f"Content-Type: text/plain; charset=utf-8\r\n" + f"Connection: close\r\n" + f"\r\n" + f"Sorry folks.\n" + ).encode(), + ) + self.assertEqual(server.state, CONNECTING) + + def test_accept_response(self): + server = ServerConnection() + with unittest.mock.patch("email.utils.formatdate", return_value=DATE): + accept = server.accept(Connect(self.make_connect_request())) + self.assertIsInstance(accept.response, Response) + self.assertEqual(accept.response.status_code, 101) + self.assertEqual(accept.response.reason_phrase, "Switching Protocols") + self.assertEqual( + accept.response.headers, + Headers( + { + "Upgrade": "websocket", + "Connection": "Upgrade", + "Sec-WebSocket-Accept": ACCEPT, + "Date": DATE, + "Server": USER_AGENT, + } + ), + ) + self.assertIsNone(accept.response.body) + + def test_reject_response(self): + server = ServerConnection() + with unittest.mock.patch("email.utils.formatdate", return_value=DATE): + reject = server.reject(http.HTTPStatus.NOT_FOUND, "Sorry folks.\n") + self.assertIsInstance(reject.response, Response) + self.assertEqual(reject.response.status_code, 404) + self.assertEqual(reject.response.reason_phrase, "Not Found") + self.assertEqual( + reject.response.headers, + Headers( + { + "Date": DATE, + "Server": USER_AGENT, + "Content-Length": "13", + "Content-Type": "text/plain; charset=utf-8", + "Connection": "close", + } + ), + ) + self.assertEqual(reject.response.body, b"Sorry folks.\n") + + def test_basic(self): + server = ServerConnection() + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + + def test_unexpected_exception(self): + server = ServerConnection() + request = self.make_connect_request() + with unittest.mock.patch( + "websockets.server.ServerConnection.process_request", + side_effect=Exception("BOOM"), + ): + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 500) + with self.assertRaises(Exception) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "BOOM") + + def test_missing_connection(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Connection"] + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 426) + self.assertEqual(reject.response.headers["Upgrade"], "websocket") + with self.assertRaises(InvalidUpgrade) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Connection header") + + def test_invalid_connection(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Connection"] + request.headers["Connection"] = "close" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 426) + self.assertEqual(reject.response.headers["Upgrade"], "websocket") + with self.assertRaises(InvalidUpgrade) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "invalid Connection header: close") + + def test_missing_upgrade(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Upgrade"] + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 426) + self.assertEqual(reject.response.headers["Upgrade"], "websocket") + with self.assertRaises(InvalidUpgrade) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Upgrade header") + + def test_invalid_upgrade(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Upgrade"] + request.headers["Upgrade"] = "h2c" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 426) + self.assertEqual(reject.response.headers["Upgrade"], "websocket") + with self.assertRaises(InvalidUpgrade) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "invalid Upgrade header: h2c") + + def test_missing_key(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Sec-WebSocket-Key"] + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Sec-WebSocket-Key header") + + def test_multiple_key(self): + server = ServerConnection() + request = self.make_connect_request() + request.headers["Sec-WebSocket-Key"] = KEY + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), + "invalid Sec-WebSocket-Key header: " + "more than one Sec-WebSocket-Key header found", + ) + + def test_invalid_key(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Sec-WebSocket-Key"] + request.headers["Sec-WebSocket-Key"] = "not Base64 data!" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), "invalid Sec-WebSocket-Key header: not Base64 data!" + ) + + def test_truncated_key(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Sec-WebSocket-Key"] + request.headers["Sec-WebSocket-Key"] = KEY[ + :16 + ] # 12 bytes instead of 16, Base64-encoded + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), f"invalid Sec-WebSocket-Key header: {KEY[:16]}" + ) + + def test_missing_version(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Sec-WebSocket-Version"] + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Sec-WebSocket-Version header") + + def test_multiple_version(self): + server = ServerConnection() + request = self.make_connect_request() + request.headers["Sec-WebSocket-Version"] = "11" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), + "invalid Sec-WebSocket-Version header: " + "more than one Sec-WebSocket-Version header found", + ) + + def test_invalid_version(self): + server = ServerConnection() + request = self.make_connect_request() + del request.headers["Sec-WebSocket-Version"] + request.headers["Sec-WebSocket-Version"] = "11" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), "invalid Sec-WebSocket-Version header: 11" + ) + + def test_no_origin(self): + server = ServerConnection(origins=["https://example.com"]) + request = self.make_connect_request() + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 403) + with self.assertRaises(InvalidOrigin) as raised: + raise reject.exception + self.assertEqual(str(raised.exception), "missing Origin header") + + def test_origin(self): + server = ServerConnection(origins=["https://example.com"]) + request = self.make_connect_request() + request.headers["Origin"] = "https://example.com" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual(server.origin, "https://example.com") + + def test_unexpected_origin(self): + server = ServerConnection(origins=["https://example.com"]) + request = self.make_connect_request() + request.headers["Origin"] = "https://other.example.com" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 403) + with self.assertRaises(InvalidOrigin) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), "invalid Origin header: https://other.example.com" + ) + + def test_multiple_origin(self): + server = ServerConnection( + origins=["https://example.com", "https://other.example.com"] + ) + request = self.make_connect_request() + request.headers["Origin"] = "https://example.com" + request.headers["Origin"] = "https://other.example.com" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + # This is prohibited by the HTTP specification, so the return code is + # 400 Bad Request rather than 403 Forbidden. + self.assertEqual(reject.response.status_code, 400) + with self.assertRaises(InvalidHeader) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), + "invalid Origin header: more than one Origin header found", + ) + + def test_supported_origin(self): + server = ServerConnection( + origins=["https://example.com", "https://other.example.com"] + ) + request = self.make_connect_request() + request.headers["Origin"] = "https://other.example.com" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual(server.origin, "https://other.example.com") + + def test_unsupported_origin(self): + server = ServerConnection( + origins=["https://example.com", "https://other.example.com"] + ) + request = self.make_connect_request() + request.headers["Origin"] = "https://original.example.com" + reject = server.accept(Connect(request)) + + self.assertIsInstance(reject, Reject) + self.assertEqual(reject.response.status_code, 403) + with self.assertRaises(InvalidOrigin) as raised: + raise reject.exception + self.assertEqual( + str(raised.exception), "invalid Origin header: https://original.example.com" + ) + + def test_no_origin_accepted(self): + server = ServerConnection(origins=[None]) + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertIsNone(server.origin) + + def test_no_extensions(self): + server = ServerConnection() + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(server.extensions, []) + + def test_no_extension(self): + server = ServerConnection(extensions=[ServerOpExtensionFactory()]) + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(server.extensions, []) + + def test_extension(self): + server = ServerConnection(extensions=[ServerOpExtensionFactory()]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-op; op" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual( + accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op" + ) + self.assertEqual(server.extensions, [OpExtension()]) + + def test_unexpected_extension(self): + server = ServerConnection() + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-op; op" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(server.extensions, []) + + def test_unsupported_extension(self): + server = ServerConnection(extensions=[ServerRsv2ExtensionFactory()]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-op; op" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(server.extensions, []) + + def test_supported_extension_parameters(self): + server = ServerConnection(extensions=[ServerOpExtensionFactory("this")]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-op; op=this" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual( + accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op=this" + ) + self.assertEqual(server.extensions, [OpExtension("this")]) + + def test_unsupported_extension_parameters(self): + server = ServerConnection(extensions=[ServerOpExtensionFactory("this")]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(server.extensions, []) + + def test_multiple_supported_extension_parameters(self): + server = ServerConnection( + extensions=[ + ServerOpExtensionFactory("this"), + ServerOpExtensionFactory("that"), + ] + ) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual( + accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op=that" + ) + self.assertEqual(server.extensions, [OpExtension("that")]) + + def test_multiple_extensions(self): + server = ServerConnection( + extensions=[ServerOpExtensionFactory(), ServerRsv2ExtensionFactory()] + ) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-op; op" + request.headers["Sec-WebSocket-Extensions"] = "x-rsv2" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual( + accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op, x-rsv2" + ) + self.assertEqual(server.extensions, [OpExtension(), Rsv2Extension()]) + + def test_multiple_extensions_order(self): + server = ServerConnection( + extensions=[ServerOpExtensionFactory(), ServerRsv2ExtensionFactory()] + ) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Extensions"] = "x-rsv2" + request.headers["Sec-WebSocket-Extensions"] = "x-op; op" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual( + accept.response.headers["Sec-WebSocket-Extensions"], "x-rsv2, x-op; op" + ) + self.assertEqual(server.extensions, [Rsv2Extension(), OpExtension()]) + + def test_no_subprotocols(self): + server = ServerConnection() + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertIsNone(server.subprotocol) + + def test_no_subprotocol(self): + server = ServerConnection(subprotocols=["chat"]) + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertIsNone(server.subprotocol) + + def test_subprotocol(self): + server = ServerConnection(subprotocols=["chat"]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Protocol"] = "chat" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual(accept.response.headers["Sec-WebSocket-Protocol"], "chat") + self.assertEqual(server.subprotocol, "chat") + + def test_unexpected_subprotocol(self): + server = ServerConnection() + request = self.make_connect_request() + request.headers["Sec-WebSocket-Protocol"] = "chat" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertIsNone(server.subprotocol) + + def test_multiple_subprotocols(self): + server = ServerConnection(subprotocols=["superchat", "chat"]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Protocol"] = "superchat" + request.headers["Sec-WebSocket-Protocol"] = "chat" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual(accept.response.headers["Sec-WebSocket-Protocol"], "superchat") + self.assertEqual(server.subprotocol, "superchat") + + def test_supported_subprotocol(self): + server = ServerConnection(subprotocols=["superchat", "chat"]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Protocol"] = "chat" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual(accept.response.headers["Sec-WebSocket-Protocol"], "chat") + self.assertEqual(server.subprotocol, "chat") + + def test_unsupported_subprotocol(self): + server = ServerConnection(subprotocols=["superchat", "chat"]) + request = self.make_connect_request() + request.headers["Sec-WebSocket-Protocol"] = "otherchat" + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertIsNone(server.subprotocol) + + def test_extra_headers(self): + for extra_headers in [ + Headers({"X-Spam": "Eggs"}), + {"X-Spam": "Eggs"}, + [("X-Spam", "Eggs")], + lambda path, headers: Headers({"X-Spam": "Eggs"}), + lambda path, headers: {"X-Spam": "Eggs"}, + lambda path, headers: [("X-Spam", "Eggs")], + ]: + with self.subTest(extra_headers=extra_headers): + server = ServerConnection(extra_headers=extra_headers) + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual(accept.response.headers["X-Spam"], "Eggs") + + def test_extra_headers_overrides_server(self): + server = ServerConnection(extra_headers={"Server": "Other"}) + request = self.make_connect_request() + accept = server.accept(Connect(request)) + + self.assertIsInstance(accept, Accept) + self.assertEqual(accept.response.headers["Server"], "Other") diff --git a/tests/utils.py b/tests/utils.py index bbffa8649..790d25687 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import email.utils import functools import logging import os @@ -7,6 +8,9 @@ import unittest +DATE = email.utils.formatdate(usegmt=True) + + class GeneratorTestCase(unittest.TestCase): def assertGeneratorRunning(self, gen): """ From 1033db5d402ed3a241356f97d642cda0df82ce45 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 21 Jun 2020 10:48:43 +0200 Subject: [PATCH 030/104] Drop Event class. It was too thin. It didn't add any value. Using the same abstractions for connection events and wire messages is good enough for our purposes. --- src/websockets/client.py | 31 ++- src/websockets/connection.py | 31 ++- src/websockets/events.py | 27 --- src/websockets/http11.py | 3 + src/websockets/protocol.py | 4 +- src/websockets/server.py | 67 +++--- tests/test_client.py | 190 +++++++++-------- tests/test_server.py | 385 ++++++++++++++++------------------- 8 files changed, 341 insertions(+), 397 deletions(-) delete mode 100644 src/websockets/events.py diff --git a/src/websockets/client.py b/src/websockets/client.py index ec4eb88f5..50203f27c 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -5,7 +5,6 @@ from .asyncio_client import WebSocketClientProtocol, connect, unix_connect from .connection import CLIENT, CONNECTING, OPEN, Connection from .datastructures import Headers, HeadersLike, MultipleValuesError -from .events import Accept, Connect, Event, Reject from .exceptions import ( InvalidHandshake, InvalidHeader, @@ -67,9 +66,9 @@ def __init__( self.extra_headers = extra_headers self.key = generate_key() - def connect(self) -> Connect: + def connect(self) -> Request: """ - Create a Connect event to send to the server. + Create a WebSocket handshake request event to send to the server. """ headers = Headers() @@ -114,8 +113,7 @@ def connect(self) -> Connect: headers.setdefault("User-Agent", USER_AGENT) - request = Request(self.wsuri.resource_name, headers) - return Connect(request) + return Request(self.wsuri.resource_name, headers) def process_response(self, response: Response) -> None: """ @@ -153,13 +151,13 @@ def process_response(self, response: Response) -> None: try: s_w_accept = headers["Sec-WebSocket-Accept"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Accept") - except MultipleValuesError: + except KeyError as exc: + raise InvalidHeader("Sec-WebSocket-Accept") from exc + except MultipleValuesError as exc: raise InvalidHeader( "Sec-WebSocket-Accept", "more than one Sec-WebSocket-Accept header found", - ) + ) from exc if s_w_accept != accept_key(self.key): raise InvalidHeaderValue("Sec-WebSocket-Accept", s_w_accept) @@ -273,11 +271,11 @@ def process_subprotocol(self, headers: Headers) -> Optional[Subprotocol]: return subprotocol - def send_in_connecting_state(self, event: Event) -> bytes: - assert isinstance(event, Connect) - - request = event.request + def send_request(self, request: Request) -> bytes: + """ + Convert a WebSocket handshake request to bytes to send to the server. + """ logger.debug("%s > GET %s HTTP/1.1", self.side, request.path) logger.debug("%s > %r", self.side, request.headers) @@ -291,9 +289,10 @@ def parse(self) -> Generator[None, None, None]: try: self.process_response(response) except InvalidHandshake as exc: - self.events.append(Reject(response, exc)) - return + response = response._replace(exception=exc) + logger.debug("Invalid handshake", exc_info=True) else: - self.events.append(Accept(response)) self.state = OPEN + finally: + self.events.append(response) yield from super().parse() diff --git a/src/websockets/connection.py b/src/websockets/connection.py index 5789b6ea1..ac9aedd6b 100644 --- a/src/websockets/connection.py +++ b/src/websockets/connection.py @@ -1,14 +1,18 @@ import enum -from typing import Generator, Iterable, List, Tuple +from typing import Generator, List, Tuple, Union -from .events import Event from .exceptions import InvalidState +from .frames import Frame +from .http11 import Request, Response from .streams import StreamReader __all__ = ["Connection"] +Event = Union[Request, Response, Frame] + + # A WebSocket connection is either a server or a client. @@ -46,43 +50,38 @@ def __init__(self, state: State = OPEN) -> None: # Public APIs for receiving data and producing events - def receive_data(self, data: bytes) -> Tuple[Iterable[Event], bytes]: + def receive_data(self, data: bytes) -> Tuple[List[Event], List[bytes]]: self.reader.feed_data(data) return self.receive() - def receive_eof(self) -> Tuple[Iterable[Event], bytes]: + def receive_eof(self) -> Tuple[List[Event], List[bytes]]: self.reader.feed_eof() return self.receive() # Public APIs for receiving events and producing data - def send(self, event: Event) -> bytes: + def send_frame(self, frame: Frame) -> bytes: """ - Send an event to the remote endpoint. + Convert a WebSocket handshake response to bytes to send. """ - if self.state == OPEN: - raise NotImplementedError # not implemented yet - elif self.state == CONNECTING: - return self.send_in_connecting_state(event) - else: + # Defensive assertion for protocol compliance. + if self.state != OPEN: raise InvalidState( f"Cannot write to a WebSocket in the {self.state.name} state" ) + raise NotImplementedError # not implemented yet # Private APIs - def send_in_connecting_state(self, event: Event) -> bytes: - raise NotImplementedError - - def receive(self) -> Tuple[List[Event], bytes]: + def receive(self) -> Tuple[List[Event], List[bytes]]: # Run parser until more data is needed or EOF try: next(self.parser) except StopIteration: pass events, self.events = self.events, [] - return events, b"" + return events, [] def parse(self) -> Generator[None, None, None]: yield # not implemented yet diff --git a/src/websockets/events.py b/src/websockets/events.py deleted file mode 100644 index 196de9421..000000000 --- a/src/websockets/events.py +++ /dev/null @@ -1,27 +0,0 @@ -from typing import NamedTuple, Optional, Union - -from .http11 import Request, Response - - -__all__ = [ - "Accept", - "Connect", - "Event", - "Reject", -] - - -class Connect(NamedTuple): - request: Request - - -class Accept(NamedTuple): - response: Response - - -class Reject(NamedTuple): - response: Response - exception: Optional[Exception] - - -Event = Union[Connect, Accept, Reject] diff --git a/src/websockets/http11.py b/src/websockets/http11.py index e1d004881..58ee09253 100644 --- a/src/websockets/http11.py +++ b/src/websockets/http11.py @@ -127,6 +127,9 @@ class Response(NamedTuple): headers: Headers body: Optional[bytes] = None + # If processing the response triggers an exception, it's stored here. + exception: Optional[Exception] = None + @classmethod def parse( cls, diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 748c1ae66..58c4569d0 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -53,7 +53,7 @@ serialize_close, ) from .framing import Frame -from .typing import Data +from .typing import Data, Subprotocol __all__ = ["WebSocketCommonProtocol"] @@ -261,7 +261,7 @@ def __init__( # WebSocket protocol parameters. self.extensions: List[Extension] = [] - self.subprotocol: Optional[str] = None + self.subprotocol: Optional[Subprotocol] = None # The close code and reason are set when receiving a close frame or # losing the TCP connection. diff --git a/src/websockets/server.py b/src/websockets/server.py index f668ff5e7..095d9a17d 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -9,7 +9,6 @@ from .asyncio_server import WebSocketServer, WebSocketServerProtocol, serve, unix_serve from .connection import CONNECTING, OPEN, SERVER, Connection from .datastructures import Headers, HeadersLike, MultipleValuesError -from .events import Accept, Connect, Event, Reject from .exceptions import ( InvalidHandshake, InvalidHeader, @@ -69,15 +68,17 @@ def __init__( self.available_subprotocols = subprotocols self.extra_headers = extra_headers - def accept(self, connect: Connect) -> Union[Accept, Reject]: + def accept(self, request: Request) -> Response: """ - Create an ``Accept`` or ``Reject`` event to send to the client. + Create a WebSocket handshake response event to send to the client. - If the connection cannot be established, this method returns a - :class:`~websockets.events.Reject` event, which may be unexpected. + If the connection cannot be established, the response rejects the + connection, which may be unexpected. """ - request = connect.request + # TODO: when changing Request to a dataclass, set the exception + # attribute on the request rather than the Response, which will + # be semantically more correct. try: key, extensions_header, protocol_header = self.process_request(request) except InvalidOrigin as exc: @@ -85,8 +86,7 @@ def accept(self, connect: Connect) -> Union[Accept, Reject]: return self.reject( http.HTTPStatus.FORBIDDEN, f"Failed to open a WebSocket connection: {exc}.\n", - exception=exc, - ) + )._replace(exception=exc) except InvalidUpgrade as exc: logger.debug("Invalid upgrade", exc_info=True) return self.reject( @@ -98,15 +98,13 @@ def accept(self, connect: Connect) -> Union[Accept, Reject]: f"with a browser. You need a WebSocket client.\n" ), headers=Headers([("Upgrade", "websocket")]), - exception=exc, - ) + )._replace(exception=exc) except InvalidHandshake as exc: logger.debug("Invalid handshake", exc_info=True) return self.reject( http.HTTPStatus.BAD_REQUEST, f"Failed to open a WebSocket connection: {exc}.\n", - exception=exc, - ) + )._replace(exception=exc) except Exception as exc: logger.warning("Error in opening handshake", exc_info=True) return self.reject( @@ -115,8 +113,7 @@ def accept(self, connect: Connect) -> Union[Accept, Reject]: "Failed to open a WebSocket connection.\n" "See server log for more information.\n" ), - exception=exc, - ) + )._replace(exception=exc) headers = Headers() @@ -146,8 +143,7 @@ def accept(self, connect: Connect) -> Union[Accept, Reject]: headers.setdefault("Date", email.utils.formatdate(usegmt=True)) headers.setdefault("Server", USER_AGENT) - response = Response(101, "Switching Protocols", headers) - return Accept(response) + return Response(101, "Switching Protocols", headers) def process_request( self, request: Request @@ -189,29 +185,29 @@ def process_request( try: key = headers["Sec-WebSocket-Key"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Key") - except MultipleValuesError: + except KeyError as exc: + raise InvalidHeader("Sec-WebSocket-Key") from exc + except MultipleValuesError as exc: raise InvalidHeader( "Sec-WebSocket-Key", "more than one Sec-WebSocket-Key header found" - ) + ) from exc try: raw_key = base64.b64decode(key.encode(), validate=True) - except binascii.Error: - raise InvalidHeaderValue("Sec-WebSocket-Key", key) + except binascii.Error as exc: + raise InvalidHeaderValue("Sec-WebSocket-Key", key) from exc if len(raw_key) != 16: raise InvalidHeaderValue("Sec-WebSocket-Key", key) try: version = headers["Sec-WebSocket-Version"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Version") - except MultipleValuesError: + except KeyError as exc: + raise InvalidHeader("Sec-WebSocket-Version") from exc + except MultipleValuesError as exc: raise InvalidHeader( "Sec-WebSocket-Version", "more than one Sec-WebSocket-Version header found", - ) + ) from exc if version != "13": raise InvalidHeaderValue("Sec-WebSocket-Version", version) @@ -389,9 +385,9 @@ def reject( text: str, headers: Optional[Headers] = None, exception: Optional[Exception] = None, - ) -> Reject: + ) -> Response: """ - Create a ``Reject`` event to send to the client. + Create a HTTP response event to send to the client. A short plain text response is the best fallback when failing to establish a WebSocket connection. @@ -405,17 +401,16 @@ def reject( headers.setdefault("Content-Length", str(len(body))) headers.setdefault("Content-Type", "text/plain; charset=utf-8") headers.setdefault("Connection", "close") - response = Response(status.value, status.phrase, headers, body) - return Reject(response, exception) + return Response(status.value, status.phrase, headers, body) - def send_in_connecting_state(self, event: Event) -> bytes: - assert isinstance(event, (Accept, Reject)) + def send_response(self, response: Response) -> bytes: + """ + Convert a WebSocket handshake response to bytes to send to the client. - if isinstance(event, Accept): + """ + if response.status_code == 101: self.state = OPEN - response = event.response - logger.debug( "%s > HTTP/1.1 %d %s", self.side, @@ -431,5 +426,5 @@ def send_in_connecting_state(self, event: Event) -> bytes: def parse(self) -> Generator[None, None, None]: request = yield from Request.parse(self.reader.read_line) assert self.state == CONNECTING - self.events.append(Connect(request)) + self.events.append(request) yield from super().parse() diff --git a/tests/test_client.py b/tests/test_client.py index 1cf27349d..eef8eb13e 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,7 +4,6 @@ from websockets.client import * from websockets.connection import CONNECTING, OPEN from websockets.datastructures import Headers -from websockets.events import Accept, Connect, Reject from websockets.exceptions import InvalidHandshake, InvalidHeader from websockets.http import USER_AGENT from websockets.http11 import Request, Response @@ -24,9 +23,9 @@ class ConnectTests(unittest.TestCase): def test_send_connect(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("wss://example.com/test") - connect = client.connect() - self.assertIsInstance(connect, Connect) - bytes_to_send = client.send(connect) + request = client.connect() + self.assertIsInstance(request, Request) + bytes_to_send = client.send_request(request) self.assertEqual( bytes_to_send, ( @@ -44,11 +43,10 @@ def test_send_connect(self): def test_connect_request(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("wss://example.com/test") - connect = client.connect() - self.assertIsInstance(connect.request, Request) - self.assertEqual(connect.request.path, "/test") + request = client.connect() + self.assertEqual(request.path, "/test") self.assertEqual( - connect.request.headers, + request.headers, Headers( { "Host": "example.com", @@ -63,7 +61,7 @@ def test_connect_request(self): def test_path(self): client = ClientConnection("wss://example.com/endpoint?test=1") - request = client.connect().request + request = client.connect() self.assertEqual(request.path, "/endpoint?test=1") @@ -78,19 +76,19 @@ def test_port(self): ]: with self.subTest(uri=uri): client = ClientConnection(uri) - request = client.connect().request + request = client.connect() self.assertEqual(request.headers["Host"], host) def test_user_info(self): client = ClientConnection("wss://hello:iloveyou@example.com/") - request = client.connect().request + request = client.connect() self.assertEqual(request.headers["Authorization"], "Basic aGVsbG86aWxvdmV5b3U=") def test_origin(self): client = ClientConnection("wss://example.com/", origin="https://example.com") - request = client.connect().request + request = client.connect() self.assertEqual(request.headers["Origin"], "https://example.com") @@ -98,13 +96,13 @@ def test_extensions(self): client = ClientConnection( "wss://example.com/", extensions=[ClientOpExtensionFactory()] ) - request = client.connect().request + request = client.connect() self.assertEqual(request.headers["Sec-WebSocket-Extensions"], "x-op; op") def test_subprotocols(self): client = ClientConnection("wss://example.com/", subprotocols=["chat"]) - request = client.connect().request + request = client.connect() self.assertEqual(request.headers["Sec-WebSocket-Protocol"], "chat") @@ -118,7 +116,7 @@ def test_extra_headers(self): client = ClientConnection( "wss://example.com/", extra_headers=extra_headers ) - request = client.connect().request + request = client.connect() self.assertEqual(request.headers["X-Spam"], "Eggs") @@ -126,7 +124,7 @@ def test_extra_headers_overrides_user_agent(self): client = ClientConnection( "wss://example.com/", extra_headers={"User-Agent": "Other"} ) - request = client.connect().request + request = client.connect() self.assertEqual(request.headers["User-Agent"], "Other") @@ -136,7 +134,7 @@ def test_receive_accept(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [accept], bytes_to_send = client.receive_data( + [response], bytes_to_send = client.receive_data( ( f"HTTP/1.1 101 Switching Protocols\r\n" f"Upgrade: websocket\r\n" @@ -147,15 +145,15 @@ def test_receive_accept(self): f"\r\n" ).encode(), ) - self.assertIsInstance(accept, Accept) - self.assertEqual(bytes_to_send, b"") + self.assertIsInstance(response, Response) + self.assertEqual(bytes_to_send, []) self.assertEqual(client.state, OPEN) def test_receive_reject(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [reject], bytes_to_send = client.receive_data( + [response], bytes_to_send = client.receive_data( ( f"HTTP/1.1 404 Not Found\r\n" f"Date: {DATE}\r\n" @@ -167,15 +165,15 @@ def test_receive_reject(self): f"Sorry folks.\n" ).encode(), ) - self.assertIsInstance(reject, Reject) - self.assertEqual(bytes_to_send, b"") + self.assertIsInstance(response, Response) + self.assertEqual(bytes_to_send, []) self.assertEqual(client.state, CONNECTING) def test_accept_response(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [accept], _bytes_to_send = client.receive_data( + [response], _bytes_to_send = client.receive_data( ( f"HTTP/1.1 101 Switching Protocols\r\n" f"Upgrade: websocket\r\n" @@ -186,10 +184,10 @@ def test_accept_response(self): f"\r\n" ).encode(), ) - self.assertEqual(accept.response.status_code, 101) - self.assertEqual(accept.response.reason_phrase, "Switching Protocols") + self.assertEqual(response.status_code, 101) + self.assertEqual(response.reason_phrase, "Switching Protocols") self.assertEqual( - accept.response.headers, + response.headers, Headers( { "Upgrade": "websocket", @@ -200,13 +198,13 @@ def test_accept_response(self): } ), ) - self.assertIsNone(accept.response.body) + self.assertIsNone(response.body) def test_reject_response(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [reject], _bytes_to_send = client.receive_data( + [response], _bytes_to_send = client.receive_data( ( f"HTTP/1.1 404 Not Found\r\n" f"Date: {DATE}\r\n" @@ -218,10 +216,10 @@ def test_reject_response(self): f"Sorry folks.\n" ).encode(), ) - self.assertEqual(reject.response.status_code, 404) - self.assertEqual(reject.response.reason_phrase, "Not Found") + self.assertEqual(response.status_code, 404) + self.assertEqual(response.reason_phrase, "Not Found") self.assertEqual( - reject.response.headers, + response.headers, Headers( { "Date": DATE, @@ -232,10 +230,10 @@ def test_reject_response(self): } ), ) - self.assertEqual(reject.response.body, b"Sorry folks.\n") + self.assertEqual(response.body, b"Sorry folks.\n") def make_accept_response(self, client): - request = client.connect().request + request = client.connect() return Response( status_code=101, reason_phrase="Switching Protocols", @@ -253,19 +251,19 @@ def make_accept_response(self, client): def test_basic(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) def test_missing_connection(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) del response.headers["Connection"] - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Connection header") def test_invalid_connection(self): @@ -273,22 +271,22 @@ def test_invalid_connection(self): response = self.make_accept_response(client) del response.headers["Connection"] response.headers["Connection"] = "close" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "invalid Connection header: close") def test_missing_upgrade(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) del response.headers["Upgrade"] - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Upgrade header") def test_invalid_upgrade(self): @@ -296,33 +294,33 @@ def test_invalid_upgrade(self): response = self.make_accept_response(client) del response.headers["Upgrade"] response.headers["Upgrade"] = "h2c" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "invalid Upgrade header: h2c") def test_missing_accept(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) del response.headers["Sec-WebSocket-Accept"] - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Sec-WebSocket-Accept header") def test_multiple_accept(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) response.headers["Sec-WebSocket-Accept"] = ACCEPT - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Sec-WebSocket-Accept header: " @@ -334,11 +332,11 @@ def test_invalid_accept(self): response = self.make_accept_response(client) del response.headers["Sec-WebSocket-Accept"] response.headers["Sec-WebSocket-Accept"] = ACCEPT - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), f"invalid Sec-WebSocket-Accept header: {ACCEPT}" ) @@ -346,9 +344,9 @@ def test_invalid_accept(self): def test_no_extensions(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, []) def test_no_extension(self): @@ -357,9 +355,9 @@ def test_no_extension(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension()]) def test_extension(self): @@ -368,20 +366,20 @@ def test_extension(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [Rsv2Extension()]) def test_unexpected_extension(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "no extensions supported") def test_unsupported_extension(self): @@ -390,11 +388,11 @@ def test_unsupported_extension(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "Unsupported extension: name = x-op, params = [('op', None)]", @@ -406,9 +404,9 @@ def test_supported_extension_parameters(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op=this" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension("this")]) def test_unsupported_extension_parameters(self): @@ -417,11 +415,11 @@ def test_unsupported_extension_parameters(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "Unsupported extension: name = x-op, params = [('op', 'that')]", @@ -437,9 +435,9 @@ def test_multiple_supported_extension_parameters(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension("that")]) def test_multiple_extensions(self): @@ -450,9 +448,9 @@ def test_multiple_extensions(self): response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension(), Rsv2Extension()]) def test_multiple_extensions_order(self): @@ -463,45 +461,45 @@ def test_multiple_extensions_order(self): response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [Rsv2Extension(), OpExtension()]) def test_no_subprotocols(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertIsNone(client.subprotocol) def test_no_subprotocol(self): client = ClientConnection("wss://example.com/", subprotocols=["chat"]) response = self.make_accept_response(client) - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertIsNone(client.subprotocol) def test_subprotocol(self): client = ClientConnection("wss://example.com/", subprotocols=["chat"]) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "chat" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.subprotocol, "chat") def test_unexpected_subprotocol(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "chat" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "no subprotocols supported") def test_multiple_subprotocols(self): @@ -511,11 +509,11 @@ def test_multiple_subprotocols(self): response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "superchat" response.headers["Sec-WebSocket-Protocol"] = "chat" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "multiple subprotocols: superchat, chat" ) @@ -526,9 +524,9 @@ def test_supported_subprotocol(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "chat" - [accept], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(accept, Accept) + self.assertEqual(client.state, OPEN) self.assertEqual(client.subprotocol, "chat") def test_unsupported_subprotocol(self): @@ -537,9 +535,9 @@ def test_unsupported_subprotocol(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "otherchat" - [reject], _bytes_to_send = client.receive_data(response.serialize()) + [response], _bytes_to_send = client.receive_data(response.serialize()) - self.assertIsInstance(reject, Reject) + self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "unsupported subprotocol: otherchat") diff --git a/tests/test_server.py b/tests/test_server.py index 1d094a86d..8b00cec11 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -4,7 +4,6 @@ from websockets.connection import CONNECTING, OPEN from websockets.datastructures import Headers -from websockets.events import Accept, Connect, Reject from websockets.exceptions import InvalidHeader, InvalidOrigin, InvalidUpgrade from websockets.http import USER_AGENT from websockets.http11 import Request, Response @@ -23,7 +22,7 @@ class ConnectTests(unittest.TestCase): def test_receive_connect(self): server = ServerConnection() - [connect], bytes_to_send = server.receive_data( + [request], bytes_to_send = server.receive_data( ( f"GET /test HTTP/1.1\r\n" f"Host: example.com\r\n" @@ -35,12 +34,12 @@ def test_receive_connect(self): f"\r\n" ).encode(), ) - self.assertIsInstance(connect, Connect) - self.assertEqual(bytes_to_send, b"") + self.assertIsInstance(request, Request) + self.assertEqual(bytes_to_send, []) def test_connect_request(self): server = ServerConnection() - [connect], bytes_to_send = server.receive_data( + [request], bytes_to_send = server.receive_data( ( f"GET /test HTTP/1.1\r\n" f"Host: example.com\r\n" @@ -52,9 +51,9 @@ def test_connect_request(self): f"\r\n" ).encode(), ) - self.assertEqual(connect.request.path, "/test") + self.assertEqual(request.path, "/test") self.assertEqual( - connect.request.headers, + request.headers, Headers( { "Host": "example.com", @@ -69,7 +68,7 @@ def test_connect_request(self): class AcceptRejectTests(unittest.TestCase): - def make_connect_request(self): + def make_request(self): return Request( path="/test", headers=Headers( @@ -87,9 +86,9 @@ def make_connect_request(self): def test_send_accept(self): server = ServerConnection() with unittest.mock.patch("email.utils.formatdate", return_value=DATE): - accept = server.accept(Connect(self.make_connect_request())) - self.assertIsInstance(accept, Accept) - bytes_to_send = server.send(accept) + response = server.accept(self.make_request()) + self.assertIsInstance(response, Response) + bytes_to_send = server.send_response(response) self.assertEqual( bytes_to_send, ( @@ -107,9 +106,9 @@ def test_send_accept(self): def test_send_reject(self): server = ServerConnection() with unittest.mock.patch("email.utils.formatdate", return_value=DATE): - reject = server.reject(http.HTTPStatus.NOT_FOUND, "Sorry folks.\n") - self.assertIsInstance(reject, Reject) - bytes_to_send = server.send(reject) + response = server.reject(http.HTTPStatus.NOT_FOUND, "Sorry folks.\n") + self.assertIsInstance(response, Response) + bytes_to_send = server.send_response(response) self.assertEqual( bytes_to_send, ( @@ -128,12 +127,12 @@ def test_send_reject(self): def test_accept_response(self): server = ServerConnection() with unittest.mock.patch("email.utils.formatdate", return_value=DATE): - accept = server.accept(Connect(self.make_connect_request())) - self.assertIsInstance(accept.response, Response) - self.assertEqual(accept.response.status_code, 101) - self.assertEqual(accept.response.reason_phrase, "Switching Protocols") + response = server.accept(self.make_request()) + self.assertIsInstance(response, Response) + self.assertEqual(response.status_code, 101) + self.assertEqual(response.reason_phrase, "Switching Protocols") self.assertEqual( - accept.response.headers, + response.headers, Headers( { "Upgrade": "websocket", @@ -144,17 +143,17 @@ def test_accept_response(self): } ), ) - self.assertIsNone(accept.response.body) + self.assertIsNone(response.body) def test_reject_response(self): server = ServerConnection() with unittest.mock.patch("email.utils.formatdate", return_value=DATE): - reject = server.reject(http.HTTPStatus.NOT_FOUND, "Sorry folks.\n") - self.assertIsInstance(reject.response, Response) - self.assertEqual(reject.response.status_code, 404) - self.assertEqual(reject.response.reason_phrase, "Not Found") + response = server.reject(http.HTTPStatus.NOT_FOUND, "Sorry folks.\n") + self.assertIsInstance(response, Response) + self.assertEqual(response.status_code, 404) + self.assertEqual(response.reason_phrase, "Not Found") self.assertEqual( - reject.response.headers, + response.headers, Headers( { "Date": DATE, @@ -165,106 +164,99 @@ def test_reject_response(self): } ), ) - self.assertEqual(reject.response.body, b"Sorry folks.\n") + self.assertEqual(response.body, b"Sorry folks.\n") def test_basic(self): server = ServerConnection() - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) + self.assertEqual(response.status_code, 101) def test_unexpected_exception(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() with unittest.mock.patch( "websockets.server.ServerConnection.process_request", side_effect=Exception("BOOM"), ): - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 500) + self.assertEqual(response.status_code, 500) with self.assertRaises(Exception) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "BOOM") def test_missing_connection(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Connection"] - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 426) - self.assertEqual(reject.response.headers["Upgrade"], "websocket") + self.assertEqual(response.status_code, 426) + self.assertEqual(response.headers["Upgrade"], "websocket") with self.assertRaises(InvalidUpgrade) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Connection header") def test_invalid_connection(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Connection"] request.headers["Connection"] = "close" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 426) - self.assertEqual(reject.response.headers["Upgrade"], "websocket") + self.assertEqual(response.status_code, 426) + self.assertEqual(response.headers["Upgrade"], "websocket") with self.assertRaises(InvalidUpgrade) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "invalid Connection header: close") def test_missing_upgrade(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Upgrade"] - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 426) - self.assertEqual(reject.response.headers["Upgrade"], "websocket") + self.assertEqual(response.status_code, 426) + self.assertEqual(response.headers["Upgrade"], "websocket") with self.assertRaises(InvalidUpgrade) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Upgrade header") def test_invalid_upgrade(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Upgrade"] request.headers["Upgrade"] = "h2c" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 426) - self.assertEqual(reject.response.headers["Upgrade"], "websocket") + self.assertEqual(response.status_code, 426) + self.assertEqual(response.headers["Upgrade"], "websocket") with self.assertRaises(InvalidUpgrade) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "invalid Upgrade header: h2c") def test_missing_key(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Sec-WebSocket-Key"] - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Sec-WebSocket-Key header") def test_multiple_key(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Key"] = KEY - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Sec-WebSocket-Key header: " @@ -273,58 +265,54 @@ def test_multiple_key(self): def test_invalid_key(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Sec-WebSocket-Key"] request.headers["Sec-WebSocket-Key"] = "not Base64 data!" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Sec-WebSocket-Key header: not Base64 data!" ) def test_truncated_key(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Sec-WebSocket-Key"] request.headers["Sec-WebSocket-Key"] = KEY[ :16 ] # 12 bytes instead of 16, Base64-encoded - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), f"invalid Sec-WebSocket-Key header: {KEY[:16]}" ) def test_missing_version(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Sec-WebSocket-Version"] - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Sec-WebSocket-Version header") def test_multiple_version(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Version"] = "11" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Sec-WebSocket-Version header: " @@ -333,49 +321,46 @@ def test_multiple_version(self): def test_invalid_version(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() del request.headers["Sec-WebSocket-Version"] request.headers["Sec-WebSocket-Version"] = "11" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Sec-WebSocket-Version header: 11" ) def test_no_origin(self): server = ServerConnection(origins=["https://example.com"]) - request = self.make_connect_request() - reject = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 403) + self.assertEqual(response.status_code, 403) with self.assertRaises(InvalidOrigin) as raised: - raise reject.exception + raise response.exception self.assertEqual(str(raised.exception), "missing Origin header") def test_origin(self): server = ServerConnection(origins=["https://example.com"]) - request = self.make_connect_request() + request = self.make_request() request.headers["Origin"] = "https://example.com" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) + self.assertEqual(response.status_code, 101) self.assertEqual(server.origin, "https://example.com") def test_unexpected_origin(self): server = ServerConnection(origins=["https://example.com"]) - request = self.make_connect_request() + request = self.make_request() request.headers["Origin"] = "https://other.example.com" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 403) + self.assertEqual(response.status_code, 403) with self.assertRaises(InvalidOrigin) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Origin header: https://other.example.com" ) @@ -384,17 +369,16 @@ def test_multiple_origin(self): server = ServerConnection( origins=["https://example.com", "https://other.example.com"] ) - request = self.make_connect_request() + request = self.make_request() request.headers["Origin"] = "https://example.com" request.headers["Origin"] = "https://other.example.com" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) # This is prohibited by the HTTP specification, so the return code is # 400 Bad Request rather than 403 Forbidden. - self.assertEqual(reject.response.status_code, 400) + self.assertEqual(response.status_code, 400) with self.assertRaises(InvalidHeader) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Origin header: more than one Origin header found", @@ -404,107 +388,102 @@ def test_supported_origin(self): server = ServerConnection( origins=["https://example.com", "https://other.example.com"] ) - request = self.make_connect_request() + request = self.make_request() request.headers["Origin"] = "https://other.example.com" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) + self.assertEqual(response.status_code, 101) self.assertEqual(server.origin, "https://other.example.com") def test_unsupported_origin(self): server = ServerConnection( origins=["https://example.com", "https://other.example.com"] ) - request = self.make_connect_request() + request = self.make_request() request.headers["Origin"] = "https://original.example.com" - reject = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(reject, Reject) - self.assertEqual(reject.response.status_code, 403) + self.assertEqual(response.status_code, 403) with self.assertRaises(InvalidOrigin) as raised: - raise reject.exception + raise response.exception self.assertEqual( str(raised.exception), "invalid Origin header: https://original.example.com" ) def test_no_origin_accepted(self): server = ServerConnection(origins=[None]) - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) + self.assertEqual(response.status_code, 101) self.assertIsNone(server.origin) def test_no_extensions(self): server = ServerConnection() - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Extensions", response.headers) self.assertEqual(server.extensions, []) def test_no_extension(self): server = ServerConnection(extensions=[ServerOpExtensionFactory()]) - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Extensions", response.headers) self.assertEqual(server.extensions, []) def test_extension(self): server = ServerConnection(extensions=[ServerOpExtensionFactory()]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-op; op" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual( - accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op" - ) + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["Sec-WebSocket-Extensions"], "x-op; op") self.assertEqual(server.extensions, [OpExtension()]) def test_unexpected_extension(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-op; op" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Extensions", response.headers) self.assertEqual(server.extensions, []) def test_unsupported_extension(self): server = ServerConnection(extensions=[ServerRsv2ExtensionFactory()]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-op; op" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Extensions", response.headers) self.assertEqual(server.extensions, []) def test_supported_extension_parameters(self): server = ServerConnection(extensions=[ServerOpExtensionFactory("this")]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-op; op=this" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual( - accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op=this" - ) + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["Sec-WebSocket-Extensions"], "x-op; op=this") self.assertEqual(server.extensions, [OpExtension("this")]) def test_unsupported_extension_parameters(self): server = ServerConnection(extensions=[ServerOpExtensionFactory("this")]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Extensions", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Extensions", response.headers) self.assertEqual(server.extensions, []) def test_multiple_supported_extension_parameters(self): @@ -514,28 +493,26 @@ def test_multiple_supported_extension_parameters(self): ServerOpExtensionFactory("that"), ] ) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual( - accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op=that" - ) + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["Sec-WebSocket-Extensions"], "x-op; op=that") self.assertEqual(server.extensions, [OpExtension("that")]) def test_multiple_extensions(self): server = ServerConnection( extensions=[ServerOpExtensionFactory(), ServerRsv2ExtensionFactory()] ) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-op; op" request.headers["Sec-WebSocket-Extensions"] = "x-rsv2" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) + self.assertEqual(response.status_code, 101) self.assertEqual( - accept.response.headers["Sec-WebSocket-Extensions"], "x-op; op, x-rsv2" + response.headers["Sec-WebSocket-Extensions"], "x-op; op, x-rsv2" ) self.assertEqual(server.extensions, [OpExtension(), Rsv2Extension()]) @@ -543,84 +520,84 @@ def test_multiple_extensions_order(self): server = ServerConnection( extensions=[ServerOpExtensionFactory(), ServerRsv2ExtensionFactory()] ) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Extensions"] = "x-rsv2" request.headers["Sec-WebSocket-Extensions"] = "x-op; op" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) + self.assertEqual(response.status_code, 101) self.assertEqual( - accept.response.headers["Sec-WebSocket-Extensions"], "x-rsv2, x-op; op" + response.headers["Sec-WebSocket-Extensions"], "x-rsv2, x-op; op" ) self.assertEqual(server.extensions, [Rsv2Extension(), OpExtension()]) def test_no_subprotocols(self): server = ServerConnection() - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Protocol", response.headers) self.assertIsNone(server.subprotocol) def test_no_subprotocol(self): server = ServerConnection(subprotocols=["chat"]) - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Protocol", response.headers) self.assertIsNone(server.subprotocol) def test_subprotocol(self): server = ServerConnection(subprotocols=["chat"]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Protocol"] = "chat" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual(accept.response.headers["Sec-WebSocket-Protocol"], "chat") + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["Sec-WebSocket-Protocol"], "chat") self.assertEqual(server.subprotocol, "chat") def test_unexpected_subprotocol(self): server = ServerConnection() - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Protocol"] = "chat" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Protocol", response.headers) self.assertIsNone(server.subprotocol) def test_multiple_subprotocols(self): server = ServerConnection(subprotocols=["superchat", "chat"]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Protocol"] = "superchat" request.headers["Sec-WebSocket-Protocol"] = "chat" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual(accept.response.headers["Sec-WebSocket-Protocol"], "superchat") + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["Sec-WebSocket-Protocol"], "superchat") self.assertEqual(server.subprotocol, "superchat") def test_supported_subprotocol(self): server = ServerConnection(subprotocols=["superchat", "chat"]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Protocol"] = "chat" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual(accept.response.headers["Sec-WebSocket-Protocol"], "chat") + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["Sec-WebSocket-Protocol"], "chat") self.assertEqual(server.subprotocol, "chat") def test_unsupported_subprotocol(self): server = ServerConnection(subprotocols=["superchat", "chat"]) - request = self.make_connect_request() + request = self.make_request() request.headers["Sec-WebSocket-Protocol"] = "otherchat" - accept = server.accept(Connect(request)) + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertNotIn("Sec-WebSocket-Protocol", accept.response.headers) + self.assertEqual(response.status_code, 101) + self.assertNotIn("Sec-WebSocket-Protocol", response.headers) self.assertIsNone(server.subprotocol) def test_extra_headers(self): @@ -634,16 +611,16 @@ def test_extra_headers(self): ]: with self.subTest(extra_headers=extra_headers): server = ServerConnection(extra_headers=extra_headers) - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual(accept.response.headers["X-Spam"], "Eggs") + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["X-Spam"], "Eggs") def test_extra_headers_overrides_server(self): server = ServerConnection(extra_headers={"Server": "Other"}) - request = self.make_connect_request() - accept = server.accept(Connect(request)) + request = self.make_request() + response = server.accept(request) - self.assertIsInstance(accept, Accept) - self.assertEqual(accept.response.headers["Server"], "Other") + self.assertEqual(response.status_code, 101) + self.assertEqual(response.headers["Server"], "Other") From f9177126eb6a6266c58345714ba75fdffd428802 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 11:50:50 +0200 Subject: [PATCH 031/104] Change Sans I/O model to handle exceptions. In the new model, receive_data returns nothing and raises an exception on errors. Events received and bytes to send are obtained through other method calls. --- src/websockets/client.py | 16 +++--- src/websockets/connection.py | 48 ++++++++++++++--- src/websockets/server.py | 23 ++++++--- tests/test_client.py | 99 +++++++++++++++++++++++------------- tests/test_server.py | 27 +++++----- 5 files changed, 143 insertions(+), 70 deletions(-) diff --git a/src/websockets/client.py b/src/websockets/client.py index 50203f27c..d6250c7e9 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -1,6 +1,6 @@ import collections import logging -from typing import Generator, List, Optional, Sequence +from typing import Any, Generator, List, Optional, Sequence from .asyncio_client import WebSocketClientProtocol, connect, unix_connect from .connection import CLIENT, CONNECTING, OPEN, Connection @@ -47,9 +47,6 @@ class ClientConnection(Connection): - - side = CLIENT - def __init__( self, uri: str, @@ -57,8 +54,9 @@ def __init__( extensions: Optional[Sequence[ClientExtensionFactory]] = None, subprotocols: Optional[Sequence[Subprotocol]] = None, extra_headers: Optional[HeadersLike] = None, + **kwargs: Any, ): - super().__init__(state=CONNECTING) + super().__init__(side=CLIENT, state=CONNECTING, **kwargs) self.wsuri = parse_uri(uri) self.origin = origin self.available_extensions = extensions @@ -271,15 +269,15 @@ def process_subprotocol(self, headers: Headers) -> Optional[Subprotocol]: return subprotocol - def send_request(self, request: Request) -> bytes: + def send_request(self, request: Request) -> None: """ - Convert a WebSocket handshake request to bytes to send to the server. + Send a WebSocket handshake request to the server. """ logger.debug("%s > GET %s HTTP/1.1", self.side, request.path) logger.debug("%s > %r", self.side, request.headers) - return request.serialize() + self.writes.append(request.serialize()) def parse(self) -> Generator[None, None, None]: response = yield from Response.parse( @@ -292,7 +290,7 @@ def parse(self) -> Generator[None, None, None]: response = response._replace(exception=exc) logger.debug("Invalid handshake", exc_info=True) else: - self.state = OPEN + self.set_state(OPEN) finally: self.events.append(response) yield from super().parse() diff --git a/src/websockets/connection.py b/src/websockets/connection.py index ac9aedd6b..616f2b3c2 100644 --- a/src/websockets/connection.py +++ b/src/websockets/connection.py @@ -1,5 +1,5 @@ import enum -from typing import Generator, List, Tuple, Union +from typing import Any, Generator, List, Tuple, Union from .exceptions import InvalidState from .frames import Frame @@ -41,22 +41,27 @@ class Connection: side: Side - def __init__(self, state: State = OPEN) -> None: + def __init__(self, side: Side, state: State = OPEN, **kwargs: Any) -> None: + self.side = side self.state = state self.reader = StreamReader() self.events: List[Event] = [] + self.writes: List[bytes] = [] self.parser = self.parse() next(self.parser) # start coroutine + def set_state(self, state: State) -> None: + self.state = state + # Public APIs for receiving data and producing events - def receive_data(self, data: bytes) -> Tuple[List[Event], List[bytes]]: + def receive_data(self, data: bytes) -> None: self.reader.feed_data(data) - return self.receive() + self.step_parser() - def receive_eof(self) -> Tuple[List[Event], List[bytes]]: + def receive_eof(self) -> None: self.reader.feed_eof() - return self.receive() + self.step_parser() # Public APIs for receiving events and producing data @@ -72,6 +77,34 @@ def send_frame(self, frame: Frame) -> bytes: ) raise NotImplementedError # not implemented yet + # Public API for getting incoming events after receiving data. + + def events_received(self) -> List[Event]: + """ + Return events read from the connection. + + Call this method immediately after calling any of the ``receive_*()`` + methods and process the events. + + """ + events, self.events = self.events, [] + return events + + # Public API for getting outgoing data after receiving data or sending events. + + def bytes_to_send(self) -> List[bytes]: + """ + Return data to write to the connection. + + Call this method immediately after calling any of the ``receive_*()`` + or ``send_*()`` methods and write the data to the connection. + + The empty bytestring signals the end of the data stream. + + """ + writes, self.writes = self.writes, [] + return writes + # Private APIs def receive(self) -> Tuple[List[Event], List[bytes]]: @@ -83,5 +116,8 @@ def receive(self) -> Tuple[List[Event], List[bytes]]: events, self.events = self.events, [] return events, [] + def step_parser(self) -> None: + next(self.parser) + def parse(self) -> Generator[None, None, None]: yield # not implemented yet diff --git a/src/websockets/server.py b/src/websockets/server.py index 095d9a17d..73156b33f 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -4,7 +4,17 @@ import email.utils import http import logging -from typing import Callable, Generator, List, Optional, Sequence, Tuple, Union, cast +from typing import ( + Any, + Callable, + Generator, + List, + Optional, + Sequence, + Tuple, + Union, + cast, +) from .asyncio_server import WebSocketServer, WebSocketServerProtocol, serve, unix_serve from .connection import CONNECTING, OPEN, SERVER, Connection @@ -61,8 +71,9 @@ def __init__( extensions: Optional[Sequence[ServerExtensionFactory]] = None, subprotocols: Optional[Sequence[Subprotocol]] = None, extra_headers: Optional[HeadersLikeOrCallable] = None, + **kwargs: Any, ): - super().__init__(state=CONNECTING) + super().__init__(SERVER, CONNECTING, **kwargs) self.origins = origins self.available_extensions = extensions self.available_subprotocols = subprotocols @@ -403,13 +414,13 @@ def reject( headers.setdefault("Connection", "close") return Response(status.value, status.phrase, headers, body) - def send_response(self, response: Response) -> bytes: + def send_response(self, response: Response) -> None: """ - Convert a WebSocket handshake response to bytes to send to the client. + Send a WebSocket handshake response to the client. """ if response.status_code == 101: - self.state = OPEN + self.set_state(OPEN) logger.debug( "%s > HTTP/1.1 %d %s", @@ -421,7 +432,7 @@ def send_response(self, response: Response) -> bytes: if response.body is not None: logger.debug("%s > body (%d bytes)", self.side, len(response.body)) - return response.serialize() + self.writes.append(response.serialize()) def parse(self) -> Generator[None, None, None]: request = yield from Request.parse(self.reader.read_line) diff --git a/tests/test_client.py b/tests/test_client.py index eef8eb13e..7a78ee09b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -25,10 +25,10 @@ def test_send_connect(self): client = ClientConnection("wss://example.com/test") request = client.connect() self.assertIsInstance(request, Request) - bytes_to_send = client.send_request(request) + client.send_request(request) self.assertEqual( - bytes_to_send, - ( + client.bytes_to_send(), + [ f"GET /test HTTP/1.1\r\n" f"Host: example.com\r\n" f"Upgrade: websocket\r\n" @@ -36,8 +36,8 @@ def test_send_connect(self): f"Sec-WebSocket-Key: {KEY}\r\n" f"Sec-WebSocket-Version: 13\r\n" f"User-Agent: {USER_AGENT}\r\n" - f"\r\n" - ).encode(), + f"\r\n".encode() + ], ) def test_connect_request(self): @@ -134,7 +134,7 @@ def test_receive_accept(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [response], bytes_to_send = client.receive_data( + client.receive_data( ( f"HTTP/1.1 101 Switching Protocols\r\n" f"Upgrade: websocket\r\n" @@ -145,15 +145,15 @@ def test_receive_accept(self): f"\r\n" ).encode(), ) + [response] = client.events_received() self.assertIsInstance(response, Response) - self.assertEqual(bytes_to_send, []) self.assertEqual(client.state, OPEN) def test_receive_reject(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [response], bytes_to_send = client.receive_data( + client.receive_data( ( f"HTTP/1.1 404 Not Found\r\n" f"Date: {DATE}\r\n" @@ -165,15 +165,15 @@ def test_receive_reject(self): f"Sorry folks.\n" ).encode(), ) + [response] = client.events_received() self.assertIsInstance(response, Response) - self.assertEqual(bytes_to_send, []) self.assertEqual(client.state, CONNECTING) def test_accept_response(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [response], _bytes_to_send = client.receive_data( + client.receive_data( ( f"HTTP/1.1 101 Switching Protocols\r\n" f"Upgrade: websocket\r\n" @@ -184,6 +184,7 @@ def test_accept_response(self): f"\r\n" ).encode(), ) + [response] = client.events_received() self.assertEqual(response.status_code, 101) self.assertEqual(response.reason_phrase, "Switching Protocols") self.assertEqual( @@ -204,7 +205,7 @@ def test_reject_response(self): with unittest.mock.patch("websockets.client.generate_key", return_value=KEY): client = ClientConnection("ws://example.com/test") client.connect() - [response], _bytes_to_send = client.receive_data( + client.receive_data( ( f"HTTP/1.1 404 Not Found\r\n" f"Date: {DATE}\r\n" @@ -216,6 +217,7 @@ def test_reject_response(self): f"Sorry folks.\n" ).encode(), ) + [response] = client.events_received() self.assertEqual(response.status_code, 404) self.assertEqual(response.reason_phrase, "Not Found") self.assertEqual( @@ -251,7 +253,8 @@ def make_accept_response(self, client): def test_basic(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) @@ -259,7 +262,8 @@ def test_missing_connection(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) del response.headers["Connection"] - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: @@ -271,7 +275,8 @@ def test_invalid_connection(self): response = self.make_accept_response(client) del response.headers["Connection"] response.headers["Connection"] = "close" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: @@ -282,7 +287,8 @@ def test_missing_upgrade(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) del response.headers["Upgrade"] - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: @@ -294,7 +300,8 @@ def test_invalid_upgrade(self): response = self.make_accept_response(client) del response.headers["Upgrade"] response.headers["Upgrade"] = "h2c" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: @@ -305,7 +312,8 @@ def test_missing_accept(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) del response.headers["Sec-WebSocket-Accept"] - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: @@ -316,7 +324,8 @@ def test_multiple_accept(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) response.headers["Sec-WebSocket-Accept"] = ACCEPT - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: @@ -332,7 +341,8 @@ def test_invalid_accept(self): response = self.make_accept_response(client) del response.headers["Sec-WebSocket-Accept"] response.headers["Sec-WebSocket-Accept"] = ACCEPT - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHeader) as raised: @@ -344,7 +354,8 @@ def test_invalid_accept(self): def test_no_extensions(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, []) @@ -355,7 +366,8 @@ def test_no_extension(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension()]) @@ -366,7 +378,8 @@ def test_extension(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [Rsv2Extension()]) @@ -375,7 +388,8 @@ def test_unexpected_extension(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: @@ -388,7 +402,8 @@ def test_unsupported_extension(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: @@ -404,7 +419,8 @@ def test_supported_extension_parameters(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op=this" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension("this")]) @@ -415,7 +431,8 @@ def test_unsupported_extension_parameters(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: @@ -435,7 +452,8 @@ def test_multiple_supported_extension_parameters(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op=that" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension("that")]) @@ -448,7 +466,8 @@ def test_multiple_extensions(self): response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-op; op" response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [OpExtension(), Rsv2Extension()]) @@ -461,7 +480,8 @@ def test_multiple_extensions_order(self): response = self.make_accept_response(client) response.headers["Sec-WebSocket-Extensions"] = "x-rsv2" response.headers["Sec-WebSocket-Extensions"] = "x-op; op" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.extensions, [Rsv2Extension(), OpExtension()]) @@ -469,7 +489,8 @@ def test_multiple_extensions_order(self): def test_no_subprotocols(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertIsNone(client.subprotocol) @@ -477,7 +498,8 @@ def test_no_subprotocols(self): def test_no_subprotocol(self): client = ClientConnection("wss://example.com/", subprotocols=["chat"]) response = self.make_accept_response(client) - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertIsNone(client.subprotocol) @@ -486,7 +508,8 @@ def test_subprotocol(self): client = ClientConnection("wss://example.com/", subprotocols=["chat"]) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "chat" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.subprotocol, "chat") @@ -495,7 +518,8 @@ def test_unexpected_subprotocol(self): client = ClientConnection("wss://example.com/") response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "chat" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: @@ -509,7 +533,8 @@ def test_multiple_subprotocols(self): response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "superchat" response.headers["Sec-WebSocket-Protocol"] = "chat" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: @@ -524,7 +549,8 @@ def test_supported_subprotocol(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "chat" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, OPEN) self.assertEqual(client.subprotocol, "chat") @@ -535,7 +561,8 @@ def test_unsupported_subprotocol(self): ) response = self.make_accept_response(client) response.headers["Sec-WebSocket-Protocol"] = "otherchat" - [response], _bytes_to_send = client.receive_data(response.serialize()) + client.receive_data(response.serialize()) + [response] = client.events_received() self.assertEqual(client.state, CONNECTING) with self.assertRaises(InvalidHandshake) as raised: diff --git a/tests/test_server.py b/tests/test_server.py index 8b00cec11..a180b08e2 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -22,7 +22,7 @@ class ConnectTests(unittest.TestCase): def test_receive_connect(self): server = ServerConnection() - [request], bytes_to_send = server.receive_data( + server.receive_data( ( f"GET /test HTTP/1.1\r\n" f"Host: example.com\r\n" @@ -34,12 +34,12 @@ def test_receive_connect(self): f"\r\n" ).encode(), ) + [request] = server.events_received() self.assertIsInstance(request, Request) - self.assertEqual(bytes_to_send, []) def test_connect_request(self): server = ServerConnection() - [request], bytes_to_send = server.receive_data( + server.receive_data( ( f"GET /test HTTP/1.1\r\n" f"Host: example.com\r\n" @@ -51,6 +51,7 @@ def test_connect_request(self): f"\r\n" ).encode(), ) + [request] = server.events_received() self.assertEqual(request.path, "/test") self.assertEqual( request.headers, @@ -88,18 +89,18 @@ def test_send_accept(self): with unittest.mock.patch("email.utils.formatdate", return_value=DATE): response = server.accept(self.make_request()) self.assertIsInstance(response, Response) - bytes_to_send = server.send_response(response) + server.send_response(response) self.assertEqual( - bytes_to_send, - ( + server.bytes_to_send(), + [ f"HTTP/1.1 101 Switching Protocols\r\n" f"Upgrade: websocket\r\n" f"Connection: Upgrade\r\n" f"Sec-WebSocket-Accept: {ACCEPT}\r\n" f"Date: {DATE}\r\n" f"Server: {USER_AGENT}\r\n" - f"\r\n" - ).encode(), + f"\r\n".encode() + ], ) self.assertEqual(server.state, OPEN) @@ -108,10 +109,10 @@ def test_send_reject(self): with unittest.mock.patch("email.utils.formatdate", return_value=DATE): response = server.reject(http.HTTPStatus.NOT_FOUND, "Sorry folks.\n") self.assertIsInstance(response, Response) - bytes_to_send = server.send_response(response) + server.send_response(response) self.assertEqual( - bytes_to_send, - ( + server.bytes_to_send(), + [ f"HTTP/1.1 404 Not Found\r\n" f"Date: {DATE}\r\n" f"Server: {USER_AGENT}\r\n" @@ -119,8 +120,8 @@ def test_send_reject(self): f"Content-Type: text/plain; charset=utf-8\r\n" f"Connection: close\r\n" f"\r\n" - f"Sorry folks.\n" - ).encode(), + f"Sorry folks.\n".encode() + ], ) self.assertEqual(server.state, CONNECTING) From 207407404d2a1bfd95da040f3948892cbf17c950 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 13:30:59 +0200 Subject: [PATCH 032/104] Implement Sans-I/O data transfer. --- setup.cfg | 2 +- src/websockets/client.py | 6 +- src/websockets/connection.py | 337 +++- src/websockets/exceptions.py | 2 +- .../extensions/permessage_deflate.py | 4 +- src/websockets/frames.py | 41 +- src/websockets/framing.py | 12 +- src/websockets/protocol.py | 3 +- src/websockets/server.py | 16 +- tests/extensions/test_permessage_deflate.py | 2 +- tests/test_connection.py | 1418 +++++++++++++++++ tests/test_frames.py | 115 +- 12 files changed, 1847 insertions(+), 111 deletions(-) create mode 100644 tests/test_connection.py diff --git a/setup.cfg b/setup.cfg index 02e70cdf5..5448b0f9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ python-tag = py36.py37 license_file = LICENSE [flake8] -ignore = E731,F403,F405,W503 +ignore = E203,E731,F403,F405,W503 max-line-length = 88 [isort] diff --git a/src/websockets/client.py b/src/websockets/client.py index d6250c7e9..3f9777b94 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -1,6 +1,6 @@ import collections import logging -from typing import Any, Generator, List, Optional, Sequence +from typing import Generator, List, Optional, Sequence from .asyncio_client import WebSocketClientProtocol, connect, unix_connect from .connection import CLIENT, CONNECTING, OPEN, Connection @@ -54,9 +54,9 @@ def __init__( extensions: Optional[Sequence[ClientExtensionFactory]] = None, subprotocols: Optional[Sequence[Subprotocol]] = None, extra_headers: Optional[HeadersLike] = None, - **kwargs: Any, + max_size: Optional[int] = 2 ** 20, ): - super().__init__(side=CLIENT, state=CONNECTING, **kwargs) + super().__init__(side=CLIENT, state=CONNECTING, max_size=max_size) self.wsuri = parse_uri(uri) self.origin = origin self.available_extensions = extensions diff --git a/src/websockets/connection.py b/src/websockets/connection.py index 616f2b3c2..ac30802db 100644 --- a/src/websockets/connection.py +++ b/src/websockets/connection.py @@ -1,14 +1,33 @@ import enum -from typing import Any, Generator, List, Tuple, Union - -from .exceptions import InvalidState -from .frames import Frame +import logging +from typing import Generator, List, Optional, Union + +from .exceptions import InvalidState, PayloadTooBig, ProtocolError +from .extensions.base import Extension +from .frames import ( + OP_BINARY, + OP_CLOSE, + OP_CONT, + OP_PING, + OP_PONG, + OP_TEXT, + Frame, + parse_close, + serialize_close, +) from .http11 import Request, Response from .streams import StreamReader +from .typing import Origin, Subprotocol -__all__ = ["Connection"] +__all__ = [ + "Connection", + "Side", + "State", + "SEND_EOF", +] +logger = logging.getLogger(__name__) Event = Union[Request, Response, Frame] @@ -37,45 +56,159 @@ class State(enum.IntEnum): CLOSED = State.CLOSED -class Connection: +# Sentinel to signal that the connection should be closed. - side: Side +SEND_EOF = b"" - def __init__(self, side: Side, state: State = OPEN, **kwargs: Any) -> None: + +class Connection: + def __init__( + self, side: Side, state: State = OPEN, max_size: Optional[int] = 2 ** 20, + ) -> None: + # Connection side. CLIENT or SERVER. self.side = side + + # Connnection state. CONNECTING and CLOSED states are handled in subclasses. + logger.debug("%s - initial state: %s", self.side, state.name) self.state = state + + # Maximum size of incoming messages in bytes. + self.max_size = max_size + + # Current size of incoming message in bytes. Only set while reading a + # fragmented message i.e. a data frames with the FIN bit not set. + self.cur_size: Optional[int] = None + + # True while sending a fragmented message i.e. a data frames with the + # FIN bit not set. + self.expect_continuation_frame = False + + # WebSocket protocol parameters. + self.origin: Optional[Origin] = None + self.extensions: List[Extension] = [] + self.subprotocol: Optional[Subprotocol] = None + + # Connection state isn't enough to tell if a close frame was received: + # when this side closes the connection, state is CLOSING as soon as a + # close frame is sent, before a close frame is received. + self.close_frame_received = False + + # Close code and reason. Set when receiving a close frame or when the + # TCP connection drops. + self.close_code: int + self.close_reason: str + + # Track if send_eof() was called. + self.eof_sent = False + + # Parser state. self.reader = StreamReader() self.events: List[Event] = [] self.writes: List[bytes] = [] self.parser = self.parse() next(self.parser) # start coroutine + self.parser_exc: Optional[Exception] = None def set_state(self, state: State) -> None: + logger.debug( + "%s - state change: %s > %s", self.side, self.state.name, state.name + ) self.state = state - # Public APIs for receiving data and producing events + # Public APIs for receiving data. def receive_data(self, data: bytes) -> None: + """ + Receive data from the connection. + + After calling this method: + + - You must call :meth:`bytes_to_send` and send this data. + - You should call :meth:`events_received` and process these events. + + """ self.reader.feed_data(data) self.step_parser() def receive_eof(self) -> None: + """ + Receive the end of the data stream from the connection. + + After calling this method: + + - You must call :meth:`bytes_to_send` and send this data. + - You shouldn't call :meth:`events_received` as it won't + return any new events. + + """ self.reader.feed_eof() self.step_parser() - # Public APIs for receiving events and producing data + # Public APIs for sending events. - def send_frame(self, frame: Frame) -> bytes: + def send_continuation(self, data: bytes, fin: bool) -> None: """ - Convert a WebSocket handshake response to bytes to send. + Send a continuation frame. """ - # Defensive assertion for protocol compliance. - if self.state != OPEN: - raise InvalidState( - f"Cannot write to a WebSocket in the {self.state.name} state" - ) - raise NotImplementedError # not implemented yet + if not self.expect_continuation_frame: + raise ProtocolError("unexpected continuation frame") + self.expect_continuation_frame = not fin + self.send_frame(Frame(fin, OP_CONT, data)) + + def send_text(self, data: bytes, fin: bool = True) -> None: + """ + Send a text frame. + + """ + if self.expect_continuation_frame: + raise ProtocolError("expected a continuation frame") + self.expect_continuation_frame = not fin + self.send_frame(Frame(fin, OP_TEXT, data)) + + def send_binary(self, data: bytes, fin: bool = True) -> None: + """ + Send a binary frame. + + """ + if self.expect_continuation_frame: + raise ProtocolError("expected a continuation frame") + self.expect_continuation_frame = not fin + self.send_frame(Frame(fin, OP_BINARY, data)) + + def send_close(self, code: Optional[int] = None, reason: str = "") -> None: + """ + Send a connection close frame. + + """ + if self.expect_continuation_frame: + raise ProtocolError("expected a continuation frame") + if code is None: + if reason != "": + raise ValueError("cannot send a reason without a code") + data = b"" + else: + data = serialize_close(code, reason) + self.send_frame(Frame(True, OP_CLOSE, data)) + # send_frame() guarantees that self.state is OPEN at this point. + # 7.1.3. The WebSocket Closing Handshake is Started + self.set_state(CLOSING) + if self.side is SERVER: + self.send_eof() + + def send_ping(self, data: bytes) -> None: + """ + Send a ping frame. + + """ + self.send_frame(Frame(True, OP_PING, data)) + + def send_pong(self, data: bytes) -> None: + """ + Send a pong frame. + + """ + self.send_frame(Frame(True, OP_PONG, data)) # Public API for getting incoming events after receiving data. @@ -105,19 +238,169 @@ def bytes_to_send(self) -> List[bytes]: writes, self.writes = self.writes, [] return writes - # Private APIs + # Private APIs for receiving data. - def receive(self) -> Tuple[List[Event], List[bytes]]: + def fail_connection(self, code: int = 1006, reason: str = "") -> None: + # Send a close frame when the state is OPEN (a close frame was already + # sent if it's CLOSING), except when failing the connection because of + # an error reading from or writing to the network. + if code != 1006 and self.state is OPEN: + self.send_frame(Frame(True, OP_CLOSE, serialize_close(code, reason))) + self.set_state(CLOSING) + if not self.eof_sent: + self.send_eof() + + def step_parser(self) -> None: # Run parser until more data is needed or EOF try: next(self.parser) except StopIteration: - pass - events, self.events = self.events, [] - return events, [] - - def step_parser(self) -> None: - next(self.parser) + # This happens if receive_data() or receive_eof() is called after + # the parser raised an exception. (It cannot happen after reaching + # EOF because receive_data() or receive_eof() would fail earlier.) + assert self.parser_exc is not None + raise RuntimeError( + "cannot receive data or EOF after an error" + ) from self.parser_exc + except ProtocolError as exc: + self.fail_connection(1002, str(exc)) + self.parser_exc = exc + raise + except EOFError as exc: + self.fail_connection(1006, str(exc)) + self.parser_exc = exc + raise + except UnicodeDecodeError as exc: + self.fail_connection(1007, f"{exc.reason} at position {exc.start}") + self.parser_exc = exc + raise + except PayloadTooBig as exc: + self.fail_connection(1009, str(exc)) + self.parser_exc = exc + raise + except Exception as exc: + logger.exception("unexpected exception in parser") + # Don't include exception details, which may be security-sensitive. + self.fail_connection(1011) + self.parser_exc = exc + raise def parse(self) -> Generator[None, None, None]: - yield # not implemented yet + while True: + eof = yield from self.reader.at_eof() + if eof: + if self.close_frame_received: + if not self.eof_sent: + self.send_eof() + yield + # Once the reader reaches EOF, its feed_data/eof() methods + # raise an error, so our receive_data/eof() methods never + # call step_parser(), so the generator shouldn't resume + # executing until it's garbage collected. + raise AssertionError( + "parser shouldn't step after EOF" + ) # pragma: no cover + else: + raise EOFError("unexpected end of stream") + + if self.max_size is None: + max_size = None + elif self.cur_size is None: + max_size = self.max_size + else: + max_size = self.max_size - self.cur_size + + frame = yield from Frame.parse( + self.reader.read_exact, + mask=self.side is SERVER, + max_size=max_size, + extensions=self.extensions, + ) + + if frame.opcode is OP_TEXT or frame.opcode is OP_BINARY: + # 5.5.1 Close: "The application MUST NOT send any more data + # frames after sending a Close frame." + if self.close_frame_received: + raise ProtocolError("data frame after close frame") + + if self.cur_size is not None: + raise ProtocolError("expected a continuation frame") + if frame.fin: + self.cur_size = None + else: + self.cur_size = len(frame.data) + + elif frame.opcode is OP_CONT: + # 5.5.1 Close: "The application MUST NOT send any more data + # frames after sending a Close frame." + if self.close_frame_received: + raise ProtocolError("data frame after close frame") + + if self.cur_size is None: + raise ProtocolError("unexpected continuation frame") + if frame.fin: + self.cur_size = None + else: + self.cur_size += len(frame.data) + + elif frame.opcode is OP_PING: + # 5.5.2. Ping: "Upon receipt of a Ping frame, an endpoint MUST + # send a Pong frame in response, unless it already received a + # Close frame." + if not self.close_frame_received: + pong_frame = Frame(True, OP_PONG, frame.data) + self.send_frame(pong_frame) + + elif frame.opcode is OP_PONG: + # 5.5.3 Pong: "A response to an unsolicited Pong frame is not + # expected." + pass + + elif frame.opcode is OP_CLOSE: + self.close_frame_received = True + # 7.1.5. The WebSocket Connection Close Code + # 7.1.6. The WebSocket Connection Close Reason + self.close_code, self.close_reason = parse_close(frame.data) + + if self.cur_size is not None: + raise ProtocolError("incomplete fragmented message") + # 5.5.1 Close: "If an endpoint receives a Close frame and did + # not previously send a Close frame, the endpoint MUST send a + # Close frame in response. (When sending a Close frame in + # response, the endpoint typically echos the status code it + # received.)" + if self.state is OPEN: + # Echo the original data instead of re-serializing it with + # serialize_close() because that fails when the close frame + # is empty and parse_close() synthetizes a 1005 close code. + # The rest is identical to send_close(). + self.send_frame(Frame(True, OP_CLOSE, frame.data)) + self.set_state(CLOSING) + if self.side is SERVER: + self.send_eof() + + else: # pragma: no cover + # This can't happen because Frame.parse() validates opcodes. + raise AssertionError(f"unexpected opcode: {frame.opcode:02x}") + + self.events.append(frame) + + # Private APIs for sending events. + + def send_frame(self, frame: Frame) -> None: + # Defensive assertion for protocol compliance. + if self.state is not OPEN: + raise InvalidState( + f"cannot write to a WebSocket in the {self.state.name} state" + ) + + logger.debug("%s > %r", self.side, frame) + self.writes.append( + frame.serialize(mask=self.side is CLIENT, extensions=self.extensions) + ) + + def send_eof(self) -> None: + assert not self.eof_sent + self.eof_sent = True + logger.debug("%s > EOF", self.side) + self.writes.append(SEND_EOF) diff --git a/src/websockets/exceptions.py b/src/websockets/exceptions.py index e593f1adc..c60a3e10e 100644 --- a/src/websockets/exceptions.py +++ b/src/websockets/exceptions.py @@ -358,7 +358,7 @@ class PayloadTooBig(WebSocketException): class ProtocolError(WebSocketException): """ - Raised when the other side breaks the protocol. + Raised when a frame breaks the protocol. """ diff --git a/src/websockets/extensions/permessage_deflate.py b/src/websockets/extensions/permessage_deflate.py index f1adf8bb6..184183061 100644 --- a/src/websockets/extensions/permessage_deflate.py +++ b/src/websockets/extensions/permessage_deflate.py @@ -128,9 +128,7 @@ def decode(self, frame: Frame, *, max_size: Optional[int] = None) -> Frame: max_length = 0 if max_size is None else max_size data = self.decoder.decompress(data, max_length) if self.decoder.unconsumed_tail: - raise PayloadTooBig( - f"Uncompressed payload length exceeds size limit (? > {max_size} bytes)" - ) + raise PayloadTooBig(f"over size limit (? > {max_size} bytes)") # Allow garbage collection of the decoder if it won't be reused. if frame.fin and self.remote_no_context_takeover: diff --git a/src/websockets/frames.py b/src/websockets/frames.py index 56dcf6171..2ff9dbd91 100644 --- a/src/websockets/frames.py +++ b/src/websockets/frames.py @@ -3,6 +3,7 @@ """ +import enum import io import secrets import struct @@ -19,14 +20,15 @@ __all__ = [ - "DATA_OPCODES", - "CTRL_OPCODES", + "Opcode", "OP_CONT", "OP_TEXT", "OP_BINARY", "OP_CLOSE", "OP_PING", "OP_PONG", + "DATA_OPCODES", + "CTRL_OPCODES", "Frame", "prepare_data", "prepare_ctrl", @@ -34,8 +36,21 @@ "serialize_close", ] -DATA_OPCODES = OP_CONT, OP_TEXT, OP_BINARY = 0x00, 0x01, 0x02 -CTRL_OPCODES = OP_CLOSE, OP_PING, OP_PONG = 0x08, 0x09, 0x0A + +class Opcode(enum.IntEnum): + CONT, TEXT, BINARY = 0x00, 0x01, 0x02 + CLOSE, PING, PONG = 0x08, 0x09, 0x0A + + +OP_CONT = Opcode.CONT +OP_TEXT = Opcode.TEXT +OP_BINARY = Opcode.BINARY +OP_CLOSE = Opcode.CLOSE +OP_PING = Opcode.PING +OP_PONG = Opcode.PONG + +DATA_OPCODES = OP_CONT, OP_TEXT, OP_BINARY +CTRL_OPCODES = OP_CLOSE, OP_PING, OP_PONG # Close code that are allowed in a close frame. # Using a list optimizes `code in EXTERNAL_CLOSE_CODES`. @@ -62,7 +77,7 @@ class Frame(NamedTuple): """ fin: bool - opcode: int + opcode: Opcode data: bytes rsv1: bool = False rsv2: bool = False @@ -103,7 +118,11 @@ def parse( rsv1 = True if head1 & 0b01000000 else False rsv2 = True if head1 & 0b00100000 else False rsv3 = True if head1 & 0b00010000 else False - opcode = head1 & 0b00001111 + + try: + opcode = Opcode(head1 & 0b00001111) + except ValueError as exc: + raise ProtocolError("invalid opcode") from exc if (True if head2 & 0b10000000 else False) != mask: raise ProtocolError("incorrect masking") @@ -116,9 +135,7 @@ def parse( data = yield from read_exact(8) (length,) = struct.unpack("!Q", data) if max_size is not None and length > max_size: - raise PayloadTooBig( - f"payload length exceeds size limit ({length} > {max_size} bytes)" - ) + raise PayloadTooBig(f"over size limit ({length} > {max_size} bytes)") if mask: mask_bytes = yield from read_exact(4) @@ -209,15 +226,11 @@ def check(self) -> None: if self.rsv1 or self.rsv2 or self.rsv3: raise ProtocolError("reserved bits must be 0") - if self.opcode in DATA_OPCODES: - return - elif self.opcode in CTRL_OPCODES: + if self.opcode in CTRL_OPCODES: if len(self.data) > 125: raise ProtocolError("control frame too long") if not self.fin: raise ProtocolError("fragmented control frame") - else: - raise ProtocolError(f"invalid opcode: {self.opcode}") def prepare_data(data: Data) -> Tuple[int, bytes]: diff --git a/src/websockets/framing.py b/src/websockets/framing.py index 221afad6f..b2996d788 100644 --- a/src/websockets/framing.py +++ b/src/websockets/framing.py @@ -15,7 +15,7 @@ from typing import Any, Awaitable, Callable, Optional, Sequence from .exceptions import PayloadTooBig, ProtocolError -from .frames import Frame as NewFrame +from .frames import Frame as NewFrame, Opcode try: @@ -64,7 +64,11 @@ async def read( rsv1 = True if head1 & 0b01000000 else False rsv2 = True if head1 & 0b00100000 else False rsv3 = True if head1 & 0b00010000 else False - opcode = head1 & 0b00001111 + + try: + opcode = Opcode(head1 & 0b00001111) + except ValueError as exc: + raise ProtocolError("invalid opcode") from exc if (True if head2 & 0b10000000 else False) != mask: raise ProtocolError("incorrect masking") @@ -77,9 +81,7 @@ async def read( data = await reader(8) (length,) = struct.unpack("!Q", data) if max_size is not None and length > max_size: - raise PayloadTooBig( - f"payload length exceeds size limit ({length} > {max_size} bytes)" - ) + raise PayloadTooBig(f"over size limit ({length} > {max_size} bytes)") if mask: mask_bits = await reader(4) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 58c4569d0..2e5d95e06 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -47,6 +47,7 @@ OP_PING, OP_PONG, OP_TEXT, + Opcode, parse_close, prepare_ctrl, prepare_data, @@ -1071,7 +1072,7 @@ async def write_frame( f"Cannot write to a WebSocket in the {self.state.name} state" ) - frame = Frame(fin, opcode, data) + frame = Frame(fin, Opcode(opcode), data) logger.debug("%s > %r", self.side, frame) frame.write( self.transport.write, mask=self.is_client, extensions=self.extensions diff --git a/src/websockets/server.py b/src/websockets/server.py index 73156b33f..1b03eabee 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -4,17 +4,7 @@ import email.utils import http import logging -from typing import ( - Any, - Callable, - Generator, - List, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import Callable, Generator, List, Optional, Sequence, Tuple, Union, cast from .asyncio_server import WebSocketServer, WebSocketServerProtocol, serve, unix_serve from .connection import CONNECTING, OPEN, SERVER, Connection @@ -71,9 +61,9 @@ def __init__( extensions: Optional[Sequence[ServerExtensionFactory]] = None, subprotocols: Optional[Sequence[Subprotocol]] = None, extra_headers: Optional[HeadersLikeOrCallable] = None, - **kwargs: Any, + max_size: Optional[int] = 2 ** 20, ): - super().__init__(SERVER, CONNECTING, **kwargs) + super().__init__(side=SERVER, state=CONNECTING, max_size=max_size) self.origins = origins self.available_extensions = extensions self.available_subprotocols = subprotocols diff --git a/tests/extensions/test_permessage_deflate.py b/tests/extensions/test_permessage_deflate.py index e1193e672..f9fca1999 100644 --- a/tests/extensions/test_permessage_deflate.py +++ b/tests/extensions/test_permessage_deflate.py @@ -243,7 +243,7 @@ def test_compress_settings(self): ), ) - # Frames aren't decoded beyond max_length. + # Frames aren't decoded beyond max_size. def test_decompress_max_size(self): frame = Frame(True, OP_TEXT, ("a" * 20).encode("utf-8")) diff --git a/tests/test_connection.py b/tests/test_connection.py new file mode 100644 index 000000000..5c0f7302f --- /dev/null +++ b/tests/test_connection.py @@ -0,0 +1,1418 @@ +import unittest.mock + +from websockets.connection import * +from websockets.exceptions import InvalidState, PayloadTooBig, ProtocolError +from websockets.frames import ( + OP_BINARY, + OP_CLOSE, + OP_CONT, + OP_PING, + OP_PONG, + OP_TEXT, + Frame, + serialize_close, +) + +from .extensions.utils import Rsv2Extension +from .test_frames import FramesTestCase + + +class ConnectionTestCase(FramesTestCase): + def assertFrameSent(self, connection, frame, eof=False): + """ + Outgoing data for ``connection`` contains the given frame. + + ``frame`` may be ``None`` if no frame is expected. + + When ``eof`` is ``True``, the end of the stream is also expected. + + """ + frames_sent = [ + None + if write is SEND_EOF + else self.parse( + write, + mask=connection.side is Side.CLIENT, + extensions=connection.extensions, + ) + for write in connection.bytes_to_send() + ] + frames_expected = [] if frame is None else [frame] + if eof: + frames_expected += [None] + self.assertEqual(frames_sent, frames_expected) + + def assertFrameReceived(self, connection, frame): + """ + Incoming data for ``connection`` contains the given frame. + + ``frame`` may be ``None`` if no frame is expected. + + """ + frames_received = connection.events_received() + frames_expected = [] if frame is None else [frame] + self.assertEqual(frames_received, frames_expected) + + def assertConnectionClosing(self, connection, code=None, reason=""): + """ + Incoming data caused the "Start the WebSocket Closing Handshake" process. + + """ + close_frame = Frame( + True, OP_CLOSE, b"" if code is None else serialize_close(code, reason), + ) + # A close frame was received. + self.assertFrameReceived(connection, close_frame) + # A close frame and possibly the end of stream were sent. + self.assertFrameSent( + connection, close_frame, eof=connection.side is Side.SERVER + ) + + def assertConnectionFailing(self, connection, code=None, reason=""): + """ + Incoming data caused the "Fail the WebSocket Connection" process. + + """ + close_frame = Frame( + True, OP_CLOSE, b"" if code is None else serialize_close(code, reason), + ) + # No frame was received. + self.assertFrameReceived(connection, None) + # A close frame and the end of stream were sent. + self.assertFrameSent(connection, close_frame, eof=True) + + +class MaskingTests(ConnectionTestCase): + """ + Test frame masking. + + 5.1. Overview + + """ + + unmasked_text_frame_date = b"\x81\x04Spam" + masked_text_frame_data = b"\x81\x84\x00\xff\x00\xff\x53\x8f\x61\x92" + + def test_client_sends_masked_frame(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\xff\x00\xff"): + client.send_text(b"Spam", True) + self.assertEqual(client.bytes_to_send(), [self.masked_text_frame_data]) + + def test_server_sends_unmasked_frame(self): + server = Connection(Side.SERVER) + server.send_text(b"Spam", True) + self.assertEqual(server.bytes_to_send(), [self.unmasked_text_frame_date]) + + def test_client_receives_unmasked_frame(self): + client = Connection(Side.CLIENT) + client.receive_data(self.unmasked_text_frame_date) + self.assertFrameReceived( + client, Frame(True, OP_TEXT, b"Spam"), + ) + + def test_server_receives_masked_frame(self): + server = Connection(Side.SERVER) + server.receive_data(self.masked_text_frame_data) + self.assertFrameReceived( + server, Frame(True, OP_TEXT, b"Spam"), + ) + + def test_client_receives_masked_frame(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(self.masked_text_frame_data) + self.assertEqual(str(raised.exception), "incorrect masking") + self.assertConnectionFailing(client, 1002, "incorrect masking") + + def test_server_receives_unmasked_frame(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(self.unmasked_text_frame_date) + self.assertEqual(str(raised.exception), "incorrect masking") + self.assertConnectionFailing(server, 1002, "incorrect masking") + + +class ContinuationTests(ConnectionTestCase): + """ + Test continuation frames without text or binary frames. + + """ + + def test_client_sends_unexpected_continuation(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.send_continuation(b"", fin=False) + self.assertEqual(str(raised.exception), "unexpected continuation frame") + + def test_server_sends_unexpected_continuation(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.send_continuation(b"", fin=False) + self.assertEqual(str(raised.exception), "unexpected continuation frame") + + def test_client_receives_unexpected_continuation(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x00\x00") + self.assertEqual(str(raised.exception), "unexpected continuation frame") + self.assertConnectionFailing(client, 1002, "unexpected continuation frame") + + def test_server_receives_unexpected_continuation(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x00\x80\x00\x00\x00\x00") + self.assertEqual(str(raised.exception), "unexpected continuation frame") + self.assertConnectionFailing(server, 1002, "unexpected continuation frame") + + def test_client_sends_continuation_after_sending_close(self): + client = Connection(Side.CLIENT) + # Since it isn't possible to send a close frame in a fragmented + # message (see test_client_send_close_in_fragmented_message), in fact, + # this is the same test as test_client_sends_unexpected_continuation. + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_close(1001) + self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + with self.assertRaises(ProtocolError) as raised: + client.send_continuation(b"", fin=False) + self.assertEqual(str(raised.exception), "unexpected continuation frame") + + def test_server_sends_continuation_after_sending_close(self): + # Since it isn't possible to send a close frame in a fragmented + # message (see test_server_send_close_in_fragmented_message), in fact, + # this is the same test as test_server_sends_unexpected_continuation. + server = Connection(Side.SERVER) + server.send_close(1000) + self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + with self.assertRaises(ProtocolError) as raised: + server.send_continuation(b"", fin=False) + self.assertEqual(str(raised.exception), "unexpected continuation frame") + + def test_client_receives_continuation_after_receiving_close(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x02\x03\xe8") + self.assertConnectionClosing(client, 1000) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x00\x00") + self.assertEqual(str(raised.exception), "data frame after close frame") + + def test_server_receives_continuation_after_receiving_close(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x82\x00\x00\x00\x00\x03\xe9") + self.assertConnectionClosing(server, 1001) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x00\x80\x00\xff\x00\xff") + self.assertEqual(str(raised.exception), "data frame after close frame") + + +class TextTests(ConnectionTestCase): + """ + Test text frames and continuation frames. + + """ + + def test_client_sends_text(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_text("😀".encode()) + self.assertEqual( + client.bytes_to_send(), [b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80"] + ) + + def test_server_sends_text(self): + server = Connection(Side.SERVER) + server.send_text("😀".encode()) + self.assertEqual(server.bytes_to_send(), [b"\x81\x04\xf0\x9f\x98\x80"]) + + def test_client_receives_text(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x81\x04\xf0\x9f\x98\x80") + self.assertFrameReceived( + client, Frame(True, OP_TEXT, "😀".encode()), + ) + + def test_server_receives_text(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80") + self.assertFrameReceived( + server, Frame(True, OP_TEXT, "😀".encode()), + ) + + def test_client_receives_text_over_size_limit(self): + client = Connection(Side.CLIENT, max_size=3) + with self.assertRaises(PayloadTooBig) as raised: + client.receive_data(b"\x81\x04\xf0\x9f\x98\x80") + self.assertEqual(str(raised.exception), "over size limit (4 > 3 bytes)") + self.assertConnectionFailing(client, 1009, "over size limit (4 > 3 bytes)") + + def test_server_receives_text_over_size_limit(self): + server = Connection(Side.SERVER, max_size=3) + with self.assertRaises(PayloadTooBig) as raised: + server.receive_data(b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80") + self.assertEqual(str(raised.exception), "over size limit (4 > 3 bytes)") + self.assertConnectionFailing(server, 1009, "over size limit (4 > 3 bytes)") + + def test_client_receives_text_without_size_limit(self): + client = Connection(Side.CLIENT, max_size=None) + client.receive_data(b"\x81\x04\xf0\x9f\x98\x80") + self.assertFrameReceived( + client, Frame(True, OP_TEXT, "😀".encode()), + ) + + def test_server_receives_text_without_size_limit(self): + server = Connection(Side.SERVER, max_size=None) + server.receive_data(b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80") + self.assertFrameReceived( + server, Frame(True, OP_TEXT, "😀".encode()), + ) + + def test_client_sends_fragmented_text(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_text("😀".encode()[:2], fin=False) + self.assertEqual(client.bytes_to_send(), [b"\x01\x82\x00\x00\x00\x00\xf0\x9f"]) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_continuation("😀😀".encode()[2:6], fin=False) + self.assertEqual( + client.bytes_to_send(), [b"\x00\x84\x00\x00\x00\x00\x98\x80\xf0\x9f"] + ) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_continuation("😀".encode()[2:], fin=True) + self.assertEqual(client.bytes_to_send(), [b"\x80\x82\x00\x00\x00\x00\x98\x80"]) + + def test_server_sends_fragmented_text(self): + server = Connection(Side.SERVER) + server.send_text("😀".encode()[:2], fin=False) + self.assertEqual(server.bytes_to_send(), [b"\x01\x02\xf0\x9f"]) + server.send_continuation("😀😀".encode()[2:6], fin=False) + self.assertEqual(server.bytes_to_send(), [b"\x00\x04\x98\x80\xf0\x9f"]) + server.send_continuation("😀".encode()[2:], fin=True) + self.assertEqual(server.bytes_to_send(), [b"\x80\x02\x98\x80"]) + + def test_client_receives_fragmented_text(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x01\x02\xf0\x9f") + self.assertFrameReceived( + client, Frame(False, OP_TEXT, "😀".encode()[:2]), + ) + client.receive_data(b"\x00\x04\x98\x80\xf0\x9f") + self.assertFrameReceived( + client, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + ) + client.receive_data(b"\x80\x02\x98\x80") + self.assertFrameReceived( + client, Frame(True, OP_CONT, "😀".encode()[2:]), + ) + + def test_server_receives_fragmented_text(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x01\x82\x00\x00\x00\x00\xf0\x9f") + self.assertFrameReceived( + server, Frame(False, OP_TEXT, "😀".encode()[:2]), + ) + server.receive_data(b"\x00\x84\x00\x00\x00\x00\x98\x80\xf0\x9f") + self.assertFrameReceived( + server, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + ) + server.receive_data(b"\x80\x82\x00\x00\x00\x00\x98\x80") + self.assertFrameReceived( + server, Frame(True, OP_CONT, "😀".encode()[2:]), + ) + + def test_client_receives_fragmented_text_over_size_limit(self): + client = Connection(Side.CLIENT, max_size=3) + client.receive_data(b"\x01\x02\xf0\x9f") + self.assertFrameReceived( + client, Frame(False, OP_TEXT, "😀".encode()[:2]), + ) + with self.assertRaises(PayloadTooBig) as raised: + client.receive_data(b"\x80\x02\x98\x80") + self.assertEqual(str(raised.exception), "over size limit (2 > 1 bytes)") + self.assertConnectionFailing(client, 1009, "over size limit (2 > 1 bytes)") + + def test_server_receives_fragmented_text_over_size_limit(self): + server = Connection(Side.SERVER, max_size=3) + server.receive_data(b"\x01\x82\x00\x00\x00\x00\xf0\x9f") + self.assertFrameReceived( + server, Frame(False, OP_TEXT, "😀".encode()[:2]), + ) + with self.assertRaises(PayloadTooBig) as raised: + server.receive_data(b"\x80\x82\x00\x00\x00\x00\x98\x80") + self.assertEqual(str(raised.exception), "over size limit (2 > 1 bytes)") + self.assertConnectionFailing(server, 1009, "over size limit (2 > 1 bytes)") + + def test_client_receives_fragmented_text_without_size_limit(self): + client = Connection(Side.CLIENT, max_size=None) + client.receive_data(b"\x01\x02\xf0\x9f") + self.assertFrameReceived( + client, Frame(False, OP_TEXT, "😀".encode()[:2]), + ) + client.receive_data(b"\x00\x04\x98\x80\xf0\x9f") + self.assertFrameReceived( + client, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + ) + client.receive_data(b"\x80\x02\x98\x80") + self.assertFrameReceived( + client, Frame(True, OP_CONT, "😀".encode()[2:]), + ) + + def test_server_receives_fragmented_text_without_size_limit(self): + server = Connection(Side.SERVER, max_size=None) + server.receive_data(b"\x01\x82\x00\x00\x00\x00\xf0\x9f") + self.assertFrameReceived( + server, Frame(False, OP_TEXT, "😀".encode()[:2]), + ) + server.receive_data(b"\x00\x84\x00\x00\x00\x00\x98\x80\xf0\x9f") + self.assertFrameReceived( + server, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + ) + server.receive_data(b"\x80\x82\x00\x00\x00\x00\x98\x80") + self.assertFrameReceived( + server, Frame(True, OP_CONT, "😀".encode()[2:]), + ) + + def test_client_sends_unexpected_text(self): + client = Connection(Side.CLIENT) + client.send_text(b"", fin=False) + with self.assertRaises(ProtocolError) as raised: + client.send_text(b"", fin=False) + self.assertEqual(str(raised.exception), "expected a continuation frame") + + def test_server_sends_unexpected_text(self): + server = Connection(Side.SERVER) + server.send_text(b"", fin=False) + with self.assertRaises(ProtocolError) as raised: + server.send_text(b"", fin=False) + self.assertEqual(str(raised.exception), "expected a continuation frame") + + def test_client_receives_unexpected_text(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x01\x00") + self.assertFrameReceived( + client, Frame(False, OP_TEXT, b""), + ) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x01\x00") + self.assertEqual(str(raised.exception), "expected a continuation frame") + self.assertConnectionFailing(client, 1002, "expected a continuation frame") + + def test_server_receives_unexpected_text(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x01\x80\x00\x00\x00\x00") + self.assertFrameReceived( + server, Frame(False, OP_TEXT, b""), + ) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x01\x80\x00\x00\x00\x00") + self.assertEqual(str(raised.exception), "expected a continuation frame") + self.assertConnectionFailing(server, 1002, "expected a continuation frame") + + def test_client_sends_text_after_sending_close(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_close(1001) + self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + with self.assertRaises(InvalidState): + client.send_text(b"") + + def test_server_sends_text_after_sending_close(self): + server = Connection(Side.SERVER) + server.send_close(1000) + self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + with self.assertRaises(InvalidState): + server.send_text(b"") + + def test_client_receives_text_after_receiving_close(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x02\x03\xe8") + self.assertConnectionClosing(client, 1000) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x81\x00") + self.assertEqual(str(raised.exception), "data frame after close frame") + + def test_server_receives_text_after_receiving_close(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x82\x00\x00\x00\x00\x03\xe9") + self.assertConnectionClosing(server, 1001) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x81\x80\x00\xff\x00\xff") + self.assertEqual(str(raised.exception), "data frame after close frame") + + +class BinaryTests(ConnectionTestCase): + """ + Test binary frames and continuation frames. + + """ + + def test_client_sends_binary(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_binary(b"\x01\x02\xfe\xff") + self.assertEqual( + client.bytes_to_send(), [b"\x82\x84\x00\x00\x00\x00\x01\x02\xfe\xff"] + ) + + def test_server_sends_binary(self): + server = Connection(Side.SERVER) + server.send_binary(b"\x01\x02\xfe\xff") + self.assertEqual(server.bytes_to_send(), [b"\x82\x04\x01\x02\xfe\xff"]) + + def test_client_receives_binary(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x82\x04\x01\x02\xfe\xff") + self.assertFrameReceived( + client, Frame(True, OP_BINARY, b"\x01\x02\xfe\xff"), + ) + + def test_server_receives_binary(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x82\x84\x00\x00\x00\x00\x01\x02\xfe\xff") + self.assertFrameReceived( + server, Frame(True, OP_BINARY, b"\x01\x02\xfe\xff"), + ) + + def test_client_receives_binary_over_size_limit(self): + client = Connection(Side.CLIENT, max_size=3) + with self.assertRaises(PayloadTooBig) as raised: + client.receive_data(b"\x82\x04\x01\x02\xfe\xff") + self.assertEqual(str(raised.exception), "over size limit (4 > 3 bytes)") + self.assertConnectionFailing(client, 1009, "over size limit (4 > 3 bytes)") + + def test_server_receives_binary_over_size_limit(self): + server = Connection(Side.SERVER, max_size=3) + with self.assertRaises(PayloadTooBig) as raised: + server.receive_data(b"\x82\x84\x00\x00\x00\x00\x01\x02\xfe\xff") + self.assertEqual(str(raised.exception), "over size limit (4 > 3 bytes)") + self.assertConnectionFailing(server, 1009, "over size limit (4 > 3 bytes)") + + def test_client_sends_fragmented_binary(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_binary(b"\x01\x02", fin=False) + self.assertEqual(client.bytes_to_send(), [b"\x02\x82\x00\x00\x00\x00\x01\x02"]) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_continuation(b"\xee\xff\x01\x02", fin=False) + self.assertEqual( + client.bytes_to_send(), [b"\x00\x84\x00\x00\x00\x00\xee\xff\x01\x02"] + ) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_continuation(b"\xee\xff", fin=True) + self.assertEqual(client.bytes_to_send(), [b"\x80\x82\x00\x00\x00\x00\xee\xff"]) + + def test_server_sends_fragmented_binary(self): + server = Connection(Side.SERVER) + server.send_binary(b"\x01\x02", fin=False) + self.assertEqual(server.bytes_to_send(), [b"\x02\x02\x01\x02"]) + server.send_continuation(b"\xee\xff\x01\x02", fin=False) + self.assertEqual(server.bytes_to_send(), [b"\x00\x04\xee\xff\x01\x02"]) + server.send_continuation(b"\xee\xff", fin=True) + self.assertEqual(server.bytes_to_send(), [b"\x80\x02\xee\xff"]) + + def test_client_receives_fragmented_binary(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x02\x02\x01\x02") + self.assertFrameReceived( + client, Frame(False, OP_BINARY, b"\x01\x02"), + ) + client.receive_data(b"\x00\x04\xfe\xff\x01\x02") + self.assertFrameReceived( + client, Frame(False, OP_CONT, b"\xfe\xff\x01\x02"), + ) + client.receive_data(b"\x80\x02\xfe\xff") + self.assertFrameReceived( + client, Frame(True, OP_CONT, b"\xfe\xff"), + ) + + def test_server_receives_fragmented_binary(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x02\x82\x00\x00\x00\x00\x01\x02") + self.assertFrameReceived( + server, Frame(False, OP_BINARY, b"\x01\x02"), + ) + server.receive_data(b"\x00\x84\x00\x00\x00\x00\xee\xff\x01\x02") + self.assertFrameReceived( + server, Frame(False, OP_CONT, b"\xee\xff\x01\x02"), + ) + server.receive_data(b"\x80\x82\x00\x00\x00\x00\xfe\xff") + self.assertFrameReceived( + server, Frame(True, OP_CONT, b"\xfe\xff"), + ) + + def test_client_receives_fragmented_binary_over_size_limit(self): + client = Connection(Side.CLIENT, max_size=3) + client.receive_data(b"\x02\x02\x01\x02") + self.assertFrameReceived( + client, Frame(False, OP_BINARY, b"\x01\x02"), + ) + with self.assertRaises(PayloadTooBig) as raised: + client.receive_data(b"\x80\x02\xfe\xff") + self.assertEqual(str(raised.exception), "over size limit (2 > 1 bytes)") + self.assertConnectionFailing(client, 1009, "over size limit (2 > 1 bytes)") + + def test_server_receives_fragmented_binary_over_size_limit(self): + server = Connection(Side.SERVER, max_size=3) + server.receive_data(b"\x02\x82\x00\x00\x00\x00\x01\x02") + self.assertFrameReceived( + server, Frame(False, OP_BINARY, b"\x01\x02"), + ) + with self.assertRaises(PayloadTooBig) as raised: + server.receive_data(b"\x80\x82\x00\x00\x00\x00\xfe\xff") + self.assertEqual(str(raised.exception), "over size limit (2 > 1 bytes)") + self.assertConnectionFailing(server, 1009, "over size limit (2 > 1 bytes)") + + def test_client_sends_unexpected_binary(self): + client = Connection(Side.CLIENT) + client.send_binary(b"", fin=False) + with self.assertRaises(ProtocolError) as raised: + client.send_binary(b"", fin=False) + self.assertEqual(str(raised.exception), "expected a continuation frame") + + def test_server_sends_unexpected_binary(self): + server = Connection(Side.SERVER) + server.send_binary(b"", fin=False) + with self.assertRaises(ProtocolError) as raised: + server.send_binary(b"", fin=False) + self.assertEqual(str(raised.exception), "expected a continuation frame") + + def test_client_receives_unexpected_binary(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x02\x00") + self.assertFrameReceived( + client, Frame(False, OP_BINARY, b""), + ) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x02\x00") + self.assertEqual(str(raised.exception), "expected a continuation frame") + self.assertConnectionFailing(client, 1002, "expected a continuation frame") + + def test_server_receives_unexpected_binary(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x02\x80\x00\x00\x00\x00") + self.assertFrameReceived( + server, Frame(False, OP_BINARY, b""), + ) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x02\x80\x00\x00\x00\x00") + self.assertEqual(str(raised.exception), "expected a continuation frame") + self.assertConnectionFailing(server, 1002, "expected a continuation frame") + + def test_client_sends_binary_after_sending_close(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_close(1001) + self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + with self.assertRaises(InvalidState): + client.send_binary(b"") + + def test_server_sends_binary_after_sending_close(self): + server = Connection(Side.SERVER) + server.send_close(1000) + self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + with self.assertRaises(InvalidState): + server.send_binary(b"") + + def test_client_receives_binary_after_receiving_close(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x02\x03\xe8") + self.assertConnectionClosing(client, 1000) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x82\x00") + self.assertEqual(str(raised.exception), "data frame after close frame") + + def test_server_receives_binary_after_receiving_close(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x82\x00\x00\x00\x00\x03\xe9") + self.assertConnectionClosing(server, 1001) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x82\x80\x00\xff\x00\xff") + self.assertEqual(str(raised.exception), "data frame after close frame") + + +class CloseTests(ConnectionTestCase): + """ + Test close frames. See 5.5.1. Close in RFC 6544. + + """ + + def test_client_sends_close(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x3c\x3c\x3c\x3c"): + client.send_close() + self.assertEqual(client.bytes_to_send(), [b"\x88\x80\x3c\x3c\x3c\x3c"]) + self.assertIs(client.state, State.CLOSING) + + def test_server_sends_close(self): + server = Connection(Side.SERVER) + server.send_close() + self.assertEqual(server.bytes_to_send(), [b"\x88\x00", b""]) + self.assertIs(server.state, State.CLOSING) + + def test_client_receives_close(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x3c\x3c\x3c\x3c"): + client.receive_data(b"\x88\x00") + self.assertEqual(client.events_received(), [Frame(True, OP_CLOSE, b"")]) + self.assertEqual(client.bytes_to_send(), [b"\x88\x80\x3c\x3c\x3c\x3c"]) + self.assertIs(client.state, State.CLOSING) + + def test_server_receives_close(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x80\x3c\x3c\x3c\x3c") + self.assertEqual(server.events_received(), [Frame(True, OP_CLOSE, b"")]) + self.assertEqual(server.bytes_to_send(), [b"\x88\x00", b""]) + self.assertIs(server.state, State.CLOSING) + + def test_client_sends_close_then_receives_close(self): + # Client-initiated close handshake on the client side. + client = Connection(Side.CLIENT) + + client.send_close() + self.assertFrameReceived(client, None) + self.assertFrameSent(client, Frame(True, OP_CLOSE, b"")) + + client.receive_data(b"\x88\x00") + self.assertFrameReceived(client, Frame(True, OP_CLOSE, b"")) + self.assertFrameSent(client, None) + + client.receive_eof() + self.assertFrameReceived(client, None) + self.assertFrameSent(client, None, eof=True) + + def test_server_sends_close_then_receives_close(self): + # Server-initiated close handshake on the server side. + server = Connection(Side.SERVER) + + server.send_close() + self.assertFrameReceived(server, None) + self.assertFrameSent(server, Frame(True, OP_CLOSE, b""), eof=True) + + server.receive_data(b"\x88\x80\x3c\x3c\x3c\x3c") + self.assertFrameReceived(server, Frame(True, OP_CLOSE, b"")) + self.assertFrameSent(server, None) + + server.receive_eof() + self.assertFrameReceived(server, None) + self.assertFrameSent(server, None) + + def test_client_receives_close_then_sends_close(self): + # Server-initiated close handshake on the client side. + client = Connection(Side.CLIENT) + + client.receive_data(b"\x88\x00") + self.assertFrameReceived(client, Frame(True, OP_CLOSE, b"")) + self.assertFrameSent(client, Frame(True, OP_CLOSE, b"")) + + client.receive_eof() + self.assertFrameReceived(client, None) + self.assertFrameSent(client, None, eof=True) + + def test_server_receives_close_then_sends_close(self): + # Client-initiated close handshake on the server side. + server = Connection(Side.SERVER) + + server.receive_data(b"\x88\x80\x3c\x3c\x3c\x3c") + self.assertFrameReceived(server, Frame(True, OP_CLOSE, b"")) + self.assertFrameSent(server, Frame(True, OP_CLOSE, b""), eof=True) + + server.receive_eof() + self.assertFrameReceived(server, None) + self.assertFrameSent(server, None) + + def test_client_sends_close_with_code(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_close(1001) + self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + self.assertIs(client.state, State.CLOSING) + + def test_server_sends_close_with_code(self): + server = Connection(Side.SERVER) + server.send_close(1000) + self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + self.assertIs(server.state, State.CLOSING) + + def test_client_receives_close_with_code(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x02\x03\xe8") + self.assertConnectionClosing(client, 1000, "") + self.assertIs(client.state, State.CLOSING) + + def test_server_receives_close_with_code(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x82\x00\x00\x00\x00\x03\xe9") + self.assertConnectionClosing(server, 1001, "") + self.assertIs(server.state, State.CLOSING) + + def test_client_sends_close_with_code_and_reason(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_close(1001, "going away") + self.assertEqual( + client.bytes_to_send(), [b"\x88\x8c\x00\x00\x00\x00\x03\xe9going away"] + ) + self.assertIs(client.state, State.CLOSING) + + def test_server_sends_close_with_code_and_reason(self): + server = Connection(Side.SERVER) + server.send_close(1000, "OK") + self.assertEqual(server.bytes_to_send(), [b"\x88\x04\x03\xe8OK", b""]) + self.assertIs(server.state, State.CLOSING) + + def test_client_receives_close_with_code_and_reason(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x04\x03\xe8OK") + self.assertConnectionClosing(client, 1000, "OK") + self.assertIs(client.state, State.CLOSING) + + def test_server_receives_close_with_code_and_reason(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x8c\x00\x00\x00\x00\x03\xe9going away") + self.assertConnectionClosing(server, 1001, "going away") + self.assertIs(server.state, State.CLOSING) + + def test_client_sends_close_with_reason_only(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ValueError) as raised: + client.send_close(reason="going away") + self.assertEqual(str(raised.exception), "cannot send a reason without a code") + + def test_server_sends_close_with_reason_only(self): + server = Connection(Side.SERVER) + with self.assertRaises(ValueError) as raised: + server.send_close(reason="OK") + self.assertEqual(str(raised.exception), "cannot send a reason without a code") + + def test_client_receives_close_with_truncated_code(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x88\x01\x03") + self.assertEqual(str(raised.exception), "close frame too short") + self.assertConnectionFailing(client, 1002, "close frame too short") + self.assertIs(client.state, State.CLOSING) + + def test_server_receives_close_with_truncated_code(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x88\x81\x00\x00\x00\x00\x03") + self.assertEqual(str(raised.exception), "close frame too short") + self.assertConnectionFailing(server, 1002, "close frame too short") + self.assertIs(server.state, State.CLOSING) + + def test_client_receives_close_with_non_utf8_reason(self): + client = Connection(Side.CLIENT) + with self.assertRaises(UnicodeDecodeError) as raised: + client.receive_data(b"\x88\x04\x03\xe8\xff\xff") + self.assertEqual( + str(raised.exception), + "'utf-8' codec can't decode byte 0xff in position 0: invalid start byte", + ) + self.assertConnectionFailing(client, 1007, "invalid start byte at position 0") + self.assertIs(client.state, State.CLOSING) + + def test_server_receives_close_with_non_utf8_reason(self): + server = Connection(Side.SERVER) + with self.assertRaises(UnicodeDecodeError) as raised: + server.receive_data(b"\x88\x84\x00\x00\x00\x00\x03\xe9\xff\xff") + self.assertEqual( + str(raised.exception), + "'utf-8' codec can't decode byte 0xff in position 0: invalid start byte", + ) + self.assertConnectionFailing(server, 1007, "invalid start byte at position 0") + self.assertIs(server.state, State.CLOSING) + + +class PingTests(ConnectionTestCase): + """ + Test ping. See 5.5.2. Ping in RFC 6544. + + """ + + def test_client_sends_ping(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x44\x88\xcc"): + client.send_ping(b"") + self.assertEqual(client.bytes_to_send(), [b"\x89\x80\x00\x44\x88\xcc"]) + + def test_server_sends_ping(self): + server = Connection(Side.SERVER) + server.send_ping(b"") + self.assertEqual(server.bytes_to_send(), [b"\x89\x00"]) + + def test_client_receives_ping(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x89\x00") + self.assertFrameReceived( + client, Frame(True, OP_PING, b""), + ) + self.assertFrameSent( + client, Frame(True, OP_PONG, b""), + ) + + def test_server_receives_ping(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x89\x80\x00\x44\x88\xcc") + self.assertFrameReceived( + server, Frame(True, OP_PING, b""), + ) + self.assertFrameSent( + server, Frame(True, OP_PONG, b""), + ) + + def test_client_sends_ping_with_data(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x44\x88\xcc"): + client.send_ping(b"\x22\x66\xaa\xee") + self.assertEqual( + client.bytes_to_send(), [b"\x89\x84\x00\x44\x88\xcc\x22\x22\x22\x22"] + ) + + def test_server_sends_ping_with_data(self): + server = Connection(Side.SERVER) + server.send_ping(b"\x22\x66\xaa\xee") + self.assertEqual(server.bytes_to_send(), [b"\x89\x04\x22\x66\xaa\xee"]) + + def test_client_receives_ping_with_data(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x89\x04\x22\x66\xaa\xee") + self.assertFrameReceived( + client, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + ) + self.assertFrameSent( + client, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + ) + + def test_server_receives_ping_with_data(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x89\x84\x00\x44\x88\xcc\x22\x22\x22\x22") + self.assertFrameReceived( + server, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + ) + self.assertFrameSent( + server, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + ) + + def test_client_sends_fragmented_ping_frame(self): + client = Connection(Side.CLIENT) + # This is only possible through a private API. + with self.assertRaises(ProtocolError) as raised: + client.send_frame(Frame(False, OP_PING, b"")) + self.assertEqual(str(raised.exception), "fragmented control frame") + + def test_server_sends_fragmented_ping_frame(self): + server = Connection(Side.SERVER) + # This is only possible through a private API. + with self.assertRaises(ProtocolError) as raised: + server.send_frame(Frame(False, OP_PING, b"")) + self.assertEqual(str(raised.exception), "fragmented control frame") + + def test_client_receives_fragmented_ping_frame(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x09\x00") + self.assertEqual(str(raised.exception), "fragmented control frame") + self.assertConnectionFailing(client, 1002, "fragmented control frame") + + def test_server_receives_fragmented_ping_frame(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x09\x80\x3c\x3c\x3c\x3c") + self.assertEqual(str(raised.exception), "fragmented control frame") + self.assertConnectionFailing(server, 1002, "fragmented control frame") + + def test_client_sends_ping_after_sending_close(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_close(1001) + self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + # The spec says: "An endpoint MAY send a Ping frame any time (...) + # before the connection is closed" but websockets doesn't support + # sending a Ping frame after a Close frame. + with self.assertRaises(InvalidState) as raised: + client.send_ping(b"") + self.assertEqual( + str(raised.exception), "cannot write to a WebSocket in the CLOSING state" + ) + + def test_server_sends_ping_after_sending_close(self): + server = Connection(Side.SERVER) + server.send_close(1000) + self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + # The spec says: "An endpoint MAY send a Ping frame any time (...) + # before the connection is closed" but websockets doesn't support + # sending a Ping frame after a Close frame. + with self.assertRaises(InvalidState) as raised: + server.send_ping(b"") + self.assertEqual( + str(raised.exception), "cannot write to a WebSocket in the CLOSING state" + ) + + def test_client_receives_ping_after_receiving_close(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x02\x03\xe8") + self.assertConnectionClosing(client, 1000) + client.receive_data(b"\x89\x04\x22\x66\xaa\xee") + self.assertFrameReceived( + client, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + ) + self.assertFrameSent(client, None) + + def test_server_receives_ping_after_receiving_close(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x82\x00\x00\x00\x00\x03\xe9") + self.assertConnectionClosing(server, 1001) + server.receive_data(b"\x89\x84\x00\x44\x88\xcc\x22\x22\x22\x22") + self.assertFrameReceived( + server, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + ) + self.assertFrameSent(server, None) + + +class PongTests(ConnectionTestCase): + """ + Test pong frames. See 5.5.3. Pong in RFC 6544. + + """ + + def test_client_sends_pong(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x44\x88\xcc"): + client.send_pong(b"") + self.assertEqual(client.bytes_to_send(), [b"\x8a\x80\x00\x44\x88\xcc"]) + + def test_server_sends_pong(self): + server = Connection(Side.SERVER) + server.send_pong(b"") + self.assertEqual(server.bytes_to_send(), [b"\x8a\x00"]) + + def test_client_receives_pong(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x8a\x00") + self.assertFrameReceived( + client, Frame(True, OP_PONG, b""), + ) + + def test_server_receives_pong(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x8a\x80\x00\x44\x88\xcc") + self.assertFrameReceived( + server, Frame(True, OP_PONG, b""), + ) + + def test_client_sends_pong_with_data(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x44\x88\xcc"): + client.send_pong(b"\x22\x66\xaa\xee") + self.assertEqual( + client.bytes_to_send(), [b"\x8a\x84\x00\x44\x88\xcc\x22\x22\x22\x22"] + ) + + def test_server_sends_pong_with_data(self): + server = Connection(Side.SERVER) + server.send_pong(b"\x22\x66\xaa\xee") + self.assertEqual(server.bytes_to_send(), [b"\x8a\x04\x22\x66\xaa\xee"]) + + def test_client_receives_pong_with_data(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x8a\x04\x22\x66\xaa\xee") + self.assertFrameReceived( + client, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + ) + + def test_server_receives_pong_with_data(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x8a\x84\x00\x44\x88\xcc\x22\x22\x22\x22") + self.assertFrameReceived( + server, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + ) + + def test_client_sends_fragmented_pong_frame(self): + client = Connection(Side.CLIENT) + # This is only possible through a private API. + with self.assertRaises(ProtocolError) as raised: + client.send_frame(Frame(False, OP_PONG, b"")) + self.assertEqual(str(raised.exception), "fragmented control frame") + + def test_server_sends_fragmented_pong_frame(self): + server = Connection(Side.SERVER) + # This is only possible through a private API. + with self.assertRaises(ProtocolError) as raised: + server.send_frame(Frame(False, OP_PONG, b"")) + self.assertEqual(str(raised.exception), "fragmented control frame") + + def test_client_receives_fragmented_pong_frame(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x0a\x00") + self.assertEqual(str(raised.exception), "fragmented control frame") + self.assertConnectionFailing(client, 1002, "fragmented control frame") + + def test_server_receives_fragmented_pong_frame(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x0a\x80\x3c\x3c\x3c\x3c") + self.assertEqual(str(raised.exception), "fragmented control frame") + self.assertConnectionFailing(server, 1002, "fragmented control frame") + + def test_client_sends_pong_after_sending_close(self): + client = Connection(Side.CLIENT) + with self.enforce_mask(b"\x00\x00\x00\x00"): + client.send_close(1001) + self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + # websockets doesn't support sending a Pong frame after a Close frame. + with self.assertRaises(InvalidState): + client.send_pong(b"") + + def test_server_sends_pong_after_sending_close(self): + server = Connection(Side.SERVER) + server.send_close(1000) + self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + # websockets doesn't support sending a Pong frame after a Close frame. + with self.assertRaises(InvalidState): + server.send_pong(b"") + + def test_client_receives_pong_after_receiving_close(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x02\x03\xe8") + self.assertConnectionClosing(client, 1000) + client.receive_data(b"\x8a\x04\x22\x66\xaa\xee") + self.assertFrameReceived( + client, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + ) + + def test_server_receives_pong_after_receiving_close(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x82\x00\x00\x00\x00\x03\xe9") + self.assertConnectionClosing(server, 1001) + server.receive_data(b"\x8a\x84\x00\x44\x88\xcc\x22\x22\x22\x22") + self.assertFrameReceived( + server, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + ) + + +class FragmentationTests(ConnectionTestCase): + """ + Test message fragmentation. + + See 5.4. Fragmentation in RFC 6544. + + """ + + def test_client_send_ping_pong_in_fragmented_message(self): + client = Connection(Side.CLIENT) + client.send_text(b"Spam", fin=False) + self.assertFrameSent(client, Frame(False, OP_TEXT, b"Spam")) + client.send_ping(b"Ping") + self.assertFrameSent(client, Frame(True, OP_PING, b"Ping")) + client.send_continuation(b"Ham", fin=False) + self.assertFrameSent(client, Frame(False, OP_CONT, b"Ham")) + client.send_pong(b"Pong") + self.assertFrameSent(client, Frame(True, OP_PONG, b"Pong")) + client.send_continuation(b"Eggs", fin=True) + self.assertFrameSent(client, Frame(True, OP_CONT, b"Eggs")) + + def test_server_send_ping_pong_in_fragmented_message(self): + server = Connection(Side.SERVER) + server.send_text(b"Spam", fin=False) + self.assertFrameSent(server, Frame(False, OP_TEXT, b"Spam")) + server.send_ping(b"Ping") + self.assertFrameSent(server, Frame(True, OP_PING, b"Ping")) + server.send_continuation(b"Ham", fin=False) + self.assertFrameSent(server, Frame(False, OP_CONT, b"Ham")) + server.send_pong(b"Pong") + self.assertFrameSent(server, Frame(True, OP_PONG, b"Pong")) + server.send_continuation(b"Eggs", fin=True) + self.assertFrameSent(server, Frame(True, OP_CONT, b"Eggs")) + + def test_client_receive_ping_pong_in_fragmented_message(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x01\x04Spam") + self.assertFrameReceived( + client, Frame(False, OP_TEXT, b"Spam"), + ) + client.receive_data(b"\x89\x04Ping") + self.assertFrameReceived( + client, Frame(True, OP_PING, b"Ping"), + ) + self.assertFrameSent( + client, Frame(True, OP_PONG, b"Ping"), + ) + client.receive_data(b"\x00\x03Ham") + self.assertFrameReceived( + client, Frame(False, OP_CONT, b"Ham"), + ) + client.receive_data(b"\x8a\x04Pong") + self.assertFrameReceived( + client, Frame(True, OP_PONG, b"Pong"), + ) + client.receive_data(b"\x80\x04Eggs") + self.assertFrameReceived( + client, Frame(True, OP_CONT, b"Eggs"), + ) + + def test_server_receive_ping_pong_in_fragmented_message(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x01\x84\x00\x00\x00\x00Spam") + self.assertFrameReceived( + server, Frame(False, OP_TEXT, b"Spam"), + ) + server.receive_data(b"\x89\x84\x00\x00\x00\x00Ping") + self.assertFrameReceived( + server, Frame(True, OP_PING, b"Ping"), + ) + self.assertFrameSent( + server, Frame(True, OP_PONG, b"Ping"), + ) + server.receive_data(b"\x00\x83\x00\x00\x00\x00Ham") + self.assertFrameReceived( + server, Frame(False, OP_CONT, b"Ham"), + ) + server.receive_data(b"\x8a\x84\x00\x00\x00\x00Pong") + self.assertFrameReceived( + server, Frame(True, OP_PONG, b"Pong"), + ) + server.receive_data(b"\x80\x84\x00\x00\x00\x00Eggs") + self.assertFrameReceived( + server, Frame(True, OP_CONT, b"Eggs"), + ) + + def test_client_send_close_in_fragmented_message(self): + client = Connection(Side.CLIENT) + client.send_text(b"Spam", fin=False) + self.assertFrameSent(client, Frame(False, OP_TEXT, b"Spam")) + # The spec says: "An endpoint MUST be capable of handling control + # frames in the middle of a fragmented message." However, since the + # endpoint must not send a data frame after a close frame, a close + # frame can't be "in the middle" of a fragmented message. + with self.assertRaises(ProtocolError) as raised: + client.send_close(1001) + self.assertEqual(str(raised.exception), "expected a continuation frame") + client.send_continuation(b"Eggs", fin=True) + + def test_server_send_close_in_fragmented_message(self): + server = Connection(Side.CLIENT) + server.send_text(b"Spam", fin=False) + self.assertFrameSent(server, Frame(False, OP_TEXT, b"Spam")) + # The spec says: "An endpoint MUST be capable of handling control + # frames in the middle of a fragmented message." However, since the + # endpoint must not send a data frame after a close frame, a close + # frame can't be "in the middle" of a fragmented message. + with self.assertRaises(ProtocolError) as raised: + server.send_close(1000) + self.assertEqual(str(raised.exception), "expected a continuation frame") + + def test_client_receive_close_in_fragmented_message(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x01\x04Spam") + self.assertFrameReceived( + client, Frame(False, OP_TEXT, b"Spam"), + ) + # The spec says: "An endpoint MUST be capable of handling control + # frames in the middle of a fragmented message." However, since the + # endpoint must not send a data frame after a close frame, a close + # frame can't be "in the middle" of a fragmented message. + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\x88\x02\x03\xe8") + self.assertEqual(str(raised.exception), "incomplete fragmented message") + self.assertConnectionFailing(client, 1002, "incomplete fragmented message") + + def test_server_receive_close_in_fragmented_message(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x01\x84\x00\x00\x00\x00Spam") + self.assertFrameReceived( + server, Frame(False, OP_TEXT, b"Spam"), + ) + # The spec says: "An endpoint MUST be capable of handling control + # frames in the middle of a fragmented message." However, since the + # endpoint must not send a data frame after a close frame, a close + # frame can't be "in the middle" of a fragmented message. + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\x88\x82\x00\x00\x00\x00\x03\xe9") + self.assertEqual(str(raised.exception), "incomplete fragmented message") + self.assertConnectionFailing(server, 1002, "incomplete fragmented message") + + +class EOFTests(ConnectionTestCase): + """ + Test connection termination. + + """ + + def test_client_receives_eof(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x00") + self.assertConnectionClosing(client) + client.receive_eof() # does not raise an exception + + def test_server_receives_eof(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x80\x3c\x3c\x3c\x3c") + self.assertConnectionClosing(server) + server.receive_eof() # does not raise an exception + + def test_client_receives_eof_between_frames(self): + client = Connection(Side.CLIENT) + with self.assertRaises(EOFError) as raised: + client.receive_eof() + self.assertEqual(str(raised.exception), "unexpected end of stream") + + def test_server_receives_eof_between_frames(self): + server = Connection(Side.SERVER) + with self.assertRaises(EOFError) as raised: + server.receive_eof() + self.assertEqual(str(raised.exception), "unexpected end of stream") + + def test_client_receives_eof_inside_frame(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x81") + with self.assertRaises(EOFError) as raised: + client.receive_eof() + self.assertEqual( + str(raised.exception), "stream ends after 1 bytes, expected 2 bytes" + ) + + def test_server_receives_eof_inside_frame(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x81") + with self.assertRaises(EOFError) as raised: + server.receive_eof() + self.assertEqual( + str(raised.exception), "stream ends after 1 bytes, expected 2 bytes" + ) + + def test_client_receives_data_after_exception(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\xff\xff") + self.assertEqual(str(raised.exception), "invalid opcode") + with self.assertRaises(RuntimeError) as raised: + client.receive_data(b"\x00\x00") + self.assertEqual( + str(raised.exception), "cannot receive data or EOF after an error" + ) + + def test_server_receives_data_after_exception(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\xff\xff") + self.assertEqual(str(raised.exception), "invalid opcode") + with self.assertRaises(RuntimeError) as raised: + server.receive_data(b"\x00\x00") + self.assertEqual( + str(raised.exception), "cannot receive data or EOF after an error" + ) + + def test_client_receives_eof_after_exception(self): + client = Connection(Side.CLIENT) + with self.assertRaises(ProtocolError) as raised: + client.receive_data(b"\xff\xff") + self.assertEqual(str(raised.exception), "invalid opcode") + with self.assertRaises(RuntimeError) as raised: + client.receive_eof() + self.assertEqual( + str(raised.exception), "cannot receive data or EOF after an error" + ) + + def test_server_receives_eof_after_exception(self): + server = Connection(Side.SERVER) + with self.assertRaises(ProtocolError) as raised: + server.receive_data(b"\xff\xff") + self.assertEqual(str(raised.exception), "invalid opcode") + with self.assertRaises(RuntimeError) as raised: + server.receive_eof() + self.assertEqual( + str(raised.exception), "cannot receive data or EOF after an error" + ) + + def test_client_receives_data_after_eof(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x00") + self.assertConnectionClosing(client) + client.receive_eof() + with self.assertRaises(EOFError) as raised: + client.receive_data(b"\x88\x00") + self.assertEqual(str(raised.exception), "stream ended") + + def test_server_receives_data_after_eof(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x80\x3c\x3c\x3c\x3c") + self.assertConnectionClosing(server) + server.receive_eof() + with self.assertRaises(EOFError) as raised: + server.receive_data(b"\x88\x80\x00\x00\x00\x00") + self.assertEqual(str(raised.exception), "stream ended") + + def test_client_receives_eof_after_eof(self): + client = Connection(Side.CLIENT) + client.receive_data(b"\x88\x00") + self.assertConnectionClosing(client) + client.receive_eof() + with self.assertRaises(EOFError) as raised: + client.receive_eof() + self.assertEqual(str(raised.exception), "stream ended") + + def test_server_receives_eof_after_eof(self): + server = Connection(Side.SERVER) + server.receive_data(b"\x88\x80\x3c\x3c\x3c\x3c") + self.assertConnectionClosing(server) + server.receive_eof() + with self.assertRaises(EOFError) as raised: + server.receive_eof() + self.assertEqual(str(raised.exception), "stream ended") + + +class ErrorTests(ConnectionTestCase): + """ + Test other error cases. + + """ + + def test_client_hits_internal_error_reading_frame(self): + client = Connection(Side.CLIENT) + # This isn't supposed to happen, so we're simulating it. + with unittest.mock.patch("struct.unpack", side_effect=RuntimeError("BOOM")): + with self.assertRaises(RuntimeError) as raised: + client.receive_data(b"\x81\x00") + self.assertEqual(str(raised.exception), "BOOM") + self.assertConnectionFailing(client, 1011, "") + + def test_server_hits_internal_error_reading_frame(self): + server = Connection(Side.SERVER) + # This isn't supposed to happen, so we're simulating it. + with unittest.mock.patch("struct.unpack", side_effect=RuntimeError("BOOM")): + with self.assertRaises(RuntimeError) as raised: + server.receive_data(b"\x81\x80\x00\x00\x00\x00") + self.assertEqual(str(raised.exception), "BOOM") + self.assertConnectionFailing(server, 1011, "") + + +class ExtensionsTests(ConnectionTestCase): + """ + Test how extensions affect frames. + + """ + + def test_client_extension_encodes_frame(self): + client = Connection(Side.CLIENT) + client.extensions = [Rsv2Extension()] + with self.enforce_mask(b"\x00\x44\x88\xcc"): + client.send_ping(b"") + self.assertEqual(client.bytes_to_send(), [b"\xa9\x80\x00\x44\x88\xcc"]) + + def test_server_extension_encodes_frame(self): + server = Connection(Side.SERVER) + server.extensions = [Rsv2Extension()] + server.send_ping(b"") + self.assertEqual(server.bytes_to_send(), [b"\xa9\x00"]) + + def test_client_extension_decodes_frame(self): + client = Connection(Side.CLIENT) + client.extensions = [Rsv2Extension()] + client.receive_data(b"\xaa\x00") + self.assertEqual(client.events_received(), [Frame(True, OP_PONG, b"")]) + + def test_server_extension_decodes_frame(self): + server = Connection(Side.SERVER) + server.extensions = [Rsv2Extension()] + server.receive_data(b"\xaa\x80\x00\x44\x88\xcc") + self.assertEqual(server.events_received(), [Frame(True, OP_PONG, b"")]) diff --git a/tests/test_frames.py b/tests/test_frames.py index 37a73b2df..514fe7c54 100644 --- a/tests/test_frames.py +++ b/tests/test_frames.py @@ -9,8 +9,15 @@ from .utils import GeneratorTestCase -class FrameTests(GeneratorTestCase): - def parse(self, data, mask=False, max_size=None, extensions=None): +class FramesTestCase(GeneratorTestCase): + def enforce_mask(self, mask): + return unittest.mock.patch("secrets.token_bytes", return_value=mask) + + def parse(self, data, mask, max_size=None, extensions=None): + """ + Parse a frame from a bytestring. + + """ reader = StreamReader() reader.feed_data(data) reader.feed_eof() @@ -19,117 +26,134 @@ def parse(self, data, mask=False, max_size=None, extensions=None): ) return self.assertGeneratorReturns(parser) - def round_trip(self, data, frame, mask=False, extensions=None): + def assertFrameData(self, frame, data, mask, extensions=None): + """ + Serializing frame yields data. Parsing data yields frame. + + """ + # Compare frames first, because test failures are easier to read, + # especially when mask = True. parsed = self.parse(data, mask=mask, extensions=extensions) self.assertEqual(parsed, frame) # Make masking deterministic by reusing the same "random" mask. # This has an effect only when mask is True. mask_bytes = data[2:6] if mask else b"" - with unittest.mock.patch("secrets.token_bytes", return_value=mask_bytes): - serialized = parsed.serialize(mask=mask, extensions=extensions) + with self.enforce_mask(mask_bytes): + serialized = frame.serialize(mask=mask, extensions=extensions) self.assertEqual(serialized, data) - def test_text(self): - self.round_trip(b"\x81\x04Spam", Frame(True, OP_TEXT, b"Spam")) + +class FrameTests(FramesTestCase): + def test_text_unmasked(self): + self.assertFrameData( + Frame(True, OP_TEXT, b"Spam"), b"\x81\x04Spam", mask=False, + ) def test_text_masked(self): - self.round_trip( - b"\x81\x84\x5b\xfb\xe1\xa8\x08\x8b\x80\xc5", + self.assertFrameData( Frame(True, OP_TEXT, b"Spam"), + b"\x81\x84\x5b\xfb\xe1\xa8\x08\x8b\x80\xc5", mask=True, ) - def test_binary(self): - self.round_trip(b"\x82\x04Eggs", Frame(True, OP_BINARY, b"Eggs")) + def test_binary_unmasked(self): + self.assertFrameData( + Frame(True, OP_BINARY, b"Eggs"), b"\x82\x04Eggs", mask=False, + ) def test_binary_masked(self): - self.round_trip( - b"\x82\x84\x53\xcd\xe2\x89\x16\xaa\x85\xfa", + self.assertFrameData( Frame(True, OP_BINARY, b"Eggs"), + b"\x82\x84\x53\xcd\xe2\x89\x16\xaa\x85\xfa", mask=True, ) - def test_non_ascii_text(self): - self.round_trip( - b"\x81\x05caf\xc3\xa9", Frame(True, OP_TEXT, "café".encode("utf-8")) + def test_non_ascii_text_unmasked(self): + self.assertFrameData( + Frame(True, OP_TEXT, "café".encode("utf-8")), + b"\x81\x05caf\xc3\xa9", + mask=False, ) def test_non_ascii_text_masked(self): - self.round_trip( - b"\x81\x85\x64\xbe\xee\x7e\x07\xdf\x88\xbd\xcd", + self.assertFrameData( Frame(True, OP_TEXT, "café".encode("utf-8")), + b"\x81\x85\x64\xbe\xee\x7e\x07\xdf\x88\xbd\xcd", mask=True, ) def test_close(self): - self.round_trip(b"\x88\x00", Frame(True, OP_CLOSE, b"")) + self.assertFrameData(Frame(True, OP_CLOSE, b""), b"\x88\x00", mask=False) def test_ping(self): - self.round_trip(b"\x89\x04ping", Frame(True, OP_PING, b"ping")) + self.assertFrameData(Frame(True, OP_PING, b"ping"), b"\x89\x04ping", mask=False) def test_pong(self): - self.round_trip(b"\x8a\x04pong", Frame(True, OP_PONG, b"pong")) + self.assertFrameData(Frame(True, OP_PONG, b"pong"), b"\x8a\x04pong", mask=False) def test_long(self): - self.round_trip( - b"\x82\x7e\x00\x7e" + 126 * b"a", Frame(True, OP_BINARY, 126 * b"a") + self.assertFrameData( + Frame(True, OP_BINARY, 126 * b"a"), + b"\x82\x7e\x00\x7e" + 126 * b"a", + mask=False, ) def test_very_long(self): - self.round_trip( - b"\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x00" + 65536 * b"a", + self.assertFrameData( Frame(True, OP_BINARY, 65536 * b"a"), + b"\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x00" + 65536 * b"a", + mask=False, ) def test_payload_too_big(self): with self.assertRaises(PayloadTooBig): - self.parse(b"\x82\x7e\x04\x01" + 1025 * b"a", max_size=1024) + self.parse(b"\x82\x7e\x04\x01" + 1025 * b"a", mask=False, max_size=1024) def test_bad_reserved_bits(self): for data in [b"\xc0\x00", b"\xa0\x00", b"\x90\x00"]: with self.subTest(data=data): with self.assertRaises(ProtocolError): - self.parse(data) + self.parse(data, mask=False) def test_good_opcode(self): for opcode in list(range(0x00, 0x03)) + list(range(0x08, 0x0B)): data = bytes([0x80 | opcode, 0]) with self.subTest(data=data): - self.parse(data) # does not raise an exception + self.parse(data, mask=False) # does not raise an exception def test_bad_opcode(self): for opcode in list(range(0x03, 0x08)) + list(range(0x0B, 0x10)): data = bytes([0x80 | opcode, 0]) with self.subTest(data=data): with self.assertRaises(ProtocolError): - self.parse(data) + self.parse(data, mask=False) def test_mask_flag(self): # Mask flag correctly set. self.parse(b"\x80\x80\x00\x00\x00\x00", mask=True) # Mask flag incorrectly unset. with self.assertRaises(ProtocolError): - self.parse(b"\x80\x80\x00\x00\x00\x00") + self.parse(b"\x80\x80\x00\x00\x00\x00", mask=False) # Mask flag correctly unset. - self.parse(b"\x80\x00") + self.parse(b"\x80\x00", mask=False) # Mask flag incorrectly set. with self.assertRaises(ProtocolError): self.parse(b"\x80\x00", mask=True) def test_control_frame_max_length(self): # At maximum allowed length. - self.parse(b"\x88\x7e\x00\x7d" + 125 * b"a") + self.parse(b"\x88\x7e\x00\x7d" + 125 * b"a", mask=False) # Above maximum allowed length. with self.assertRaises(ProtocolError): - self.parse(b"\x88\x7e\x00\x7e" + 126 * b"a") + self.parse(b"\x88\x7e\x00\x7e" + 126 * b"a", mask=False) def test_fragmented_control_frame(self): # Fin bit correctly set. - self.parse(b"\x88\x00") + self.parse(b"\x88\x00", mask=False) # Fin bit incorrectly unset. with self.assertRaises(ProtocolError): - self.parse(b"\x08\x00") + self.parse(b"\x08\x00", mask=False) def test_extensions(self): class Rot13: @@ -145,8 +169,11 @@ def encode(frame): def decode(frame, *, max_size=None): return Rot13.encode(frame) - self.round_trip( - b"\x81\x05uryyb", Frame(True, OP_TEXT, b"hello"), extensions=[Rot13()] + self.assertFrameData( + Frame(True, OP_TEXT, b"hello"), + b"\x81\x05uryyb", + mask=False, + extensions=[Rot13()], ) @@ -205,15 +232,19 @@ def test_prepare_ctrl_none(self): class ParseAndSerializeCloseTests(unittest.TestCase): - def round_trip(self, data, code, reason): - parsed = parse_close(data) - self.assertEqual(parsed, (code, reason)) + def assertCloseData(self, code, reason, data): + """ + Serializing code / reason yields data. Parsing data yields code / reason. + + """ serialized = serialize_close(code, reason) self.assertEqual(serialized, data) + parsed = parse_close(data) + self.assertEqual(parsed, (code, reason)) def test_parse_close_and_serialize_close(self): - self.round_trip(b"\x03\xe8", 1000, "") - self.round_trip(b"\x03\xe8OK", 1000, "OK") + self.assertCloseData(1000, "", b"\x03\xe8") + self.assertCloseData(1000, "OK", b"\x03\xe8OK") def test_parse_close_empty(self): self.assertEqual(parse_close(b""), (1005, "")) From fad4c57d4d84cb884bd30ebe44e07ace4d5f4cfb Mon Sep 17 00:00:00 2001 From: akgnah <1024@setq.me> Date: Mon, 2 Mar 2020 13:01:51 +0800 Subject: [PATCH 033/104] fix typo in example/counter.py --- example/counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/counter.py b/example/counter.py index dbbbe5935..239ec203a 100755 --- a/example/counter.py +++ b/example/counter.py @@ -58,7 +58,7 @@ async def counter(websocket, path): STATE["value"] += 1 await notify_state() else: - logging.error("unsupported event: {}", data) + logging.error("unsupported event: %s", data) finally: await unregister(websocket) From 458c4d67faaaf52359f713aafc3eda26afb1de3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20L=C3=89VEIL?= Date: Thu, 9 Apr 2020 01:09:36 +0200 Subject: [PATCH 034/104] support request lines of 4107 bytes fix #743 avoid sending a `HTTP 400` response when popular browsers send a request with cookies maxing up the user-agent limit --- src/websockets/http11.py | 2 +- src/websockets/http_legacy.py | 2 +- tests/test_http11.py | 4 ++-- tests/test_http_legacy.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/websockets/http11.py b/src/websockets/http11.py index 58ee09253..693a20e54 100644 --- a/src/websockets/http11.py +++ b/src/websockets/http11.py @@ -6,7 +6,7 @@ MAX_HEADERS = 256 -MAX_LINE = 4096 +MAX_LINE = 4107 def d(value: bytes) -> str: diff --git a/src/websockets/http_legacy.py b/src/websockets/http_legacy.py index 3630d3593..0bc548b31 100644 --- a/src/websockets/http_legacy.py +++ b/src/websockets/http_legacy.py @@ -9,7 +9,7 @@ __all__ = ["read_request", "read_response"] MAX_HEADERS = 256 -MAX_LINE = 4096 +MAX_LINE = 4107 def d(value: bytes) -> str: diff --git a/tests/test_http11.py b/tests/test_http11.py index 4574cf97e..87be6e486 100644 --- a/tests/test_http11.py +++ b/tests/test_http11.py @@ -260,8 +260,8 @@ def test_parse_too_long_value(self): next(self.parse_headers()) def test_parse_too_long_line(self): - # Header line contains 5 + 4090 + 2 = 4097 bytes. - self.reader.feed_data(b"foo: " + b"a" * 4090 + b"\r\n\r\n") + # Header line contains 5 + 4101 + 2 = 4108 bytes. + self.reader.feed_data(b"foo: " + b"a" * 4101 + b"\r\n\r\n") with self.assertRaises(SecurityError): next(self.parse_headers()) diff --git a/tests/test_http_legacy.py b/tests/test_http_legacy.py index 3b43a6274..667aff52a 100644 --- a/tests/test_http_legacy.py +++ b/tests/test_http_legacy.py @@ -124,8 +124,8 @@ async def test_headers_limit(self): await read_headers(self.stream) async def test_line_limit(self): - # Header line contains 5 + 4090 + 2 = 4097 bytes. - self.stream.feed_data(b"foo: " + b"a" * 4090 + b"\r\n\r\n") + # Header line contains 5 + 4101 + 2 = 4108 bytes. + self.stream.feed_data(b"foo: " + b"a" * 4101 + b"\r\n\r\n") with self.assertRaises(SecurityError): await read_headers(self.stream) From f056c1cfb8ef417180bf337308aa73e49c9469b4 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 21:10:56 +0200 Subject: [PATCH 035/104] Adjust max header size (again). See #743 for the rationale. --- src/websockets/http11.py | 2 +- src/websockets/http_legacy.py | 2 +- tests/test_http11.py | 4 ++-- tests/test_http_legacy.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/websockets/http11.py b/src/websockets/http11.py index 693a20e54..0754ddabb 100644 --- a/src/websockets/http11.py +++ b/src/websockets/http11.py @@ -6,7 +6,7 @@ MAX_HEADERS = 256 -MAX_LINE = 4107 +MAX_LINE = 4110 def d(value: bytes) -> str: diff --git a/src/websockets/http_legacy.py b/src/websockets/http_legacy.py index 0bc548b31..5afe5f898 100644 --- a/src/websockets/http_legacy.py +++ b/src/websockets/http_legacy.py @@ -9,7 +9,7 @@ __all__ = ["read_request", "read_response"] MAX_HEADERS = 256 -MAX_LINE = 4107 +MAX_LINE = 4110 def d(value: bytes) -> str: diff --git a/tests/test_http11.py b/tests/test_http11.py index 87be6e486..9e4d70620 100644 --- a/tests/test_http11.py +++ b/tests/test_http11.py @@ -260,8 +260,8 @@ def test_parse_too_long_value(self): next(self.parse_headers()) def test_parse_too_long_line(self): - # Header line contains 5 + 4101 + 2 = 4108 bytes. - self.reader.feed_data(b"foo: " + b"a" * 4101 + b"\r\n\r\n") + # Header line contains 5 + 4104 + 2 = 4111 bytes. + self.reader.feed_data(b"foo: " + b"a" * 4104 + b"\r\n\r\n") with self.assertRaises(SecurityError): next(self.parse_headers()) diff --git a/tests/test_http_legacy.py b/tests/test_http_legacy.py index 667aff52a..e4c75315e 100644 --- a/tests/test_http_legacy.py +++ b/tests/test_http_legacy.py @@ -124,8 +124,8 @@ async def test_headers_limit(self): await read_headers(self.stream) async def test_line_limit(self): - # Header line contains 5 + 4101 + 2 = 4108 bytes. - self.stream.feed_data(b"foo: " + b"a" * 4101 + b"\r\n\r\n") + # Header line contains 5 + 4104 + 2 = 4111 bytes. + self.stream.feed_data(b"foo: " + b"a" * 4104 + b"\r\n\r\n") with self.assertRaises(SecurityError): await read_headers(self.stream) From 639b993a236107f22d529cde488d1e1eb6645228 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 21:38:15 +0200 Subject: [PATCH 036/104] Create correct Host header for IPv6. Fix #802. --- src/websockets/asyncio_client.py | 7 ++----- src/websockets/client.py | 9 ++++----- src/websockets/http.py | 26 +++++++++++++++++++++++++- tests/test_http.py | 29 +++++++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/websockets/asyncio_client.py b/src/websockets/asyncio_client.py index f95dae060..e01a641cb 100644 --- a/src/websockets/asyncio_client.py +++ b/src/websockets/asyncio_client.py @@ -31,7 +31,7 @@ parse_extension, parse_subprotocol, ) -from .http import USER_AGENT +from .http import USER_AGENT, build_host from .http_legacy import read_response from .protocol import WebSocketCommonProtocol from .typing import ExtensionHeader, Origin, Subprotocol @@ -251,10 +251,7 @@ async def handshake( """ request_headers = Headers() - if wsuri.port == (443 if wsuri.secure else 80): # pragma: no cover - request_headers["Host"] = wsuri.host - else: - request_headers["Host"] = f"{wsuri.host}:{wsuri.port}" + request_headers["Host"] = build_host(wsuri.host, wsuri.port, wsuri.secure) if wsuri.user_info: request_headers["Authorization"] = build_authorization_basic( diff --git a/src/websockets/client.py b/src/websockets/client.py index 3f9777b94..a7bfcc4ee 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -23,7 +23,7 @@ parse_subprotocol, parse_upgrade, ) -from .http import USER_AGENT +from .http import USER_AGENT, build_host from .http11 import Request, Response from .typing import ( ConnectionOption, @@ -71,10 +71,9 @@ def connect(self) -> Request: """ headers = Headers() - if self.wsuri.port == (443 if self.wsuri.secure else 80): - headers["Host"] = self.wsuri.host - else: - headers["Host"] = f"{self.wsuri.host}:{self.wsuri.port}" + headers["Host"] = build_host( + self.wsuri.host, self.wsuri.port, self.wsuri.secure + ) if self.wsuri.user_info: headers["Authorization"] = build_authorization_basic(*self.wsuri.user_info) diff --git a/src/websockets/http.py b/src/websockets/http.py index 850b9beaa..ed3fe48d0 100644 --- a/src/websockets/http.py +++ b/src/websockets/http.py @@ -1,4 +1,5 @@ import asyncio +import ipaddress import sys import warnings from typing import Tuple @@ -9,13 +10,36 @@ from .version import version as websockets_version -__all__ = ["USER_AGENT"] +__all__ = ["USER_AGENT", "build_host"] PYTHON_VERSION = "{}.{}".format(*sys.version_info) USER_AGENT = f"Python/{PYTHON_VERSION} websockets/{websockets_version}" +def build_host(host: str, port: int, secure: bool) -> str: + """ + Build a ``Host`` header. + + """ + # https://tools.ietf.org/html/rfc3986#section-3.2.2 + # IPv6 addresses must be enclosed in brackets. + try: + address = ipaddress.ip_address(host) + except ValueError: + # host is a hostname + pass + else: + # host is an IP address + if address.version == 6: + host = f"[{host}]" + + if port != (443 if secure else 80): + host = f"{host}:{port}" + + return host + + # Backwards compatibility with previously documented public APIs diff --git a/tests/test_http.py b/tests/test_http.py index 322650354..ca7c1c0a4 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,2 +1,27 @@ -# Check that the legacy http module imports without an exception. -from websockets.http import * # noqa +import unittest + +from websockets.http import * + + +class HTTPTests(unittest.TestCase): + def test_build_host(self): + for (host, port, secure), result in [ + (("localhost", 80, False), "localhost"), + (("localhost", 8000, False), "localhost:8000"), + (("localhost", 443, True), "localhost"), + (("localhost", 8443, True), "localhost:8443"), + (("example.com", 80, False), "example.com"), + (("example.com", 8000, False), "example.com:8000"), + (("example.com", 443, True), "example.com"), + (("example.com", 8443, True), "example.com:8443"), + (("127.0.0.1", 80, False), "127.0.0.1"), + (("127.0.0.1", 8000, False), "127.0.0.1:8000"), + (("127.0.0.1", 443, True), "127.0.0.1"), + (("127.0.0.1", 8443, True), "127.0.0.1:8443"), + (("::1", 80, False), "[::1]"), + (("::1", 8000, False), "[::1]:8000"), + (("::1", 443, True), "[::1]"), + (("::1", 8443, True), "[::1]:8443"), + ]: + with self.subTest(host=host, port=port, secure=secure): + self.assertEqual(build_host(host, port, secure), result) From 6466e238f4809e81579f70460563fa0d00b7905a Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 21:49:05 +0200 Subject: [PATCH 037/104] Raise a good error when sending a dict. This must be a common mistake. Fix #734. --- src/websockets/protocol.py | 10 ++++++++++ tests/test_protocol.py | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 2e5d95e06..92ce8e305 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -25,6 +25,7 @@ Dict, Iterable, List, + Mapping, Optional, Union, cast, @@ -548,6 +549,10 @@ async def send( :meth:`send` will raise a :exc:`TypeError` and the connection will be closed. + :meth:`send` rejects dict-like objects because this is often an error. + If you wish to send the keys of a dict-like object as fragments, call + its :meth:`~dict.keys` method and pass the result to :meth:`send`. + Canceling :meth:`send` is discouraged. Instead, you should close the connection with :meth:`close`. Indeed, there only two situations where :meth:`send` yields control to the event loop: @@ -576,6 +581,11 @@ async def send( opcode, data = prepare_data(message) await self.write_frame(True, opcode, data) + # Catch a common mistake -- passing a dict to send(). + + elif isinstance(message, Mapping): + raise TypeError("data is a dict-like object") + # Fragmented message -- regular iterator. elif isinstance(message, Iterable): diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 3054600e1..432c31ef5 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -584,6 +584,11 @@ def test_send_binary_from_non_contiguous_memoryview(self): self.loop.run_until_complete(self.protocol.send(memoryview(b"tteeaa")[::2])) self.assertOneFrameSent(True, OP_BINARY, b"tea") + def test_send_dict(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.send({"not": "encoded"})) + self.assertNoFrameSent() + def test_send_type_error(self): with self.assertRaises(TypeError): self.loop.run_until_complete(self.protocol.send(42)) From 97ae02b4560516f577b265ef222fff5fb3e950b6 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 22:05:31 +0200 Subject: [PATCH 038/104] Document pitfall. Fix #335. --- docs/faq.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index cea3f5358..5e6439055 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -204,6 +204,13 @@ There are several reasons why long-lived connections may be lost: If you're facing a reproducible issue, :ref:`enable debug logs ` to see when and how connections are closed. +Why do I get the error: ``module 'websockets' has no attribute '...'``? +....................................................................... + +Often, this is because you created a script called ``websockets.py`` in your +current working directory. Then ``import websockets`` imports this module +instead of the websockets library. + Are there ``onopen``, ``onmessage``, ``onerror``, and ``onclose`` callbacks? ............................................................................ From 0a1195eed14eddb3f27929ef49af4024814c3f37 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 26 Jul 2020 22:48:28 +0200 Subject: [PATCH 039/104] Type create_protocol arguments as callables. Fix #764. --- src/websockets/asyncio_client.py | 4 ++-- src/websockets/asyncio_server.py | 2 +- src/websockets/auth.py | 13 +++++++++---- tests/test_auth.py | 22 +++++++++++++++++++++- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/websockets/asyncio_client.py b/src/websockets/asyncio_client.py index e01a641cb..efa29b69a 100644 --- a/src/websockets/asyncio_client.py +++ b/src/websockets/asyncio_client.py @@ -9,7 +9,7 @@ import logging import warnings from types import TracebackType -from typing import Any, Generator, List, Optional, Sequence, Tuple, Type, cast +from typing import Any, Callable, Generator, List, Optional, Sequence, Tuple, Type, cast from .datastructures import Headers, HeadersLike from .exceptions import ( @@ -373,7 +373,7 @@ def __init__( uri: str, *, path: Optional[str] = None, - create_protocol: Optional[Type[WebSocketClientProtocol]] = None, + create_protocol: Optional[Callable[[Any], WebSocketClientProtocol]] = None, ping_interval: Optional[float] = 20, ping_timeout: Optional[float] = 20, close_timeout: Optional[float] = None, diff --git a/src/websockets/asyncio_server.py b/src/websockets/asyncio_server.py index 89ddf6c7d..fe61c7ddc 100644 --- a/src/websockets/asyncio_server.py +++ b/src/websockets/asyncio_server.py @@ -850,7 +850,7 @@ def __init__( port: Optional[int] = None, *, path: Optional[str] = None, - create_protocol: Optional[Type[WebSocketServerProtocol]] = None, + create_protocol: Optional[Callable[[Any], WebSocketServerProtocol]] = None, ping_interval: Optional[float] = 20, ping_timeout: Optional[float] = 20, close_timeout: Optional[float] = None, diff --git a/src/websockets/auth.py b/src/websockets/auth.py index 03e8536c5..c1b7a0b1a 100644 --- a/src/websockets/auth.py +++ b/src/websockets/auth.py @@ -7,7 +7,7 @@ import functools import http -from typing import Any, Awaitable, Callable, Iterable, Optional, Tuple, Type, Union +from typing import Any, Awaitable, Callable, Iterable, Optional, Tuple, Union, cast from .asyncio_server import HTTPResponse, WebSocketServerProtocol from .datastructures import Headers @@ -90,9 +90,7 @@ def basic_auth_protocol_factory( realm: str, credentials: Optional[Union[Credentials, Iterable[Credentials]]] = None, check_credentials: Optional[Callable[[str, str], Awaitable[bool]]] = None, - create_protocol: Type[ - BasicAuthWebSocketServerProtocol - ] = BasicAuthWebSocketServerProtocol, + create_protocol: Optional[Callable[[Any], BasicAuthWebSocketServerProtocol]] = None, ) -> Callable[[Any], BasicAuthWebSocketServerProtocol]: """ Protocol factory that enforces HTTP Basic Auth. @@ -155,6 +153,13 @@ async def check_credentials(username: str, password: str) -> bool: else: raise TypeError(f"invalid credentials argument: {credentials}") + if create_protocol is None: + # Not sure why mypy cannot figure this out. + create_protocol = cast( + Callable[[Any], BasicAuthWebSocketServerProtocol], + BasicAuthWebSocketServerProtocol, + ) + return functools.partial( create_protocol, realm=realm, check_credentials=check_credentials ) diff --git a/tests/test_auth.py b/tests/test_auth.py index c693c9f45..68642389e 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -19,6 +19,12 @@ def test_is_not_credentials(self): self.assertFalse(is_credentials("username")) +class CustomWebSocketServerProtocol(BasicAuthWebSocketServerProtocol): + async def process_request(self, path, request_headers): + type(self).used = True + return await super().process_request(path, request_headers) + + class AuthClientServerTests(ClientServerTestsMixin, AsyncioTestCase): create_protocol = basic_auth_protocol_factory( @@ -73,7 +79,7 @@ async def check_credentials(username, password): return password == "iloveyou" create_protocol_check_credentials = basic_auth_protocol_factory( - realm="auth-tests", check_credentials=check_credentials + realm="auth-tests", check_credentials=check_credentials, ) @with_server(create_protocol=create_protocol_check_credentials) @@ -82,6 +88,20 @@ def test_basic_auth_check_credentials(self): self.loop.run_until_complete(self.client.send("Hello!")) self.loop.run_until_complete(self.client.recv()) + create_protocol_custom_protocol = basic_auth_protocol_factory( + realm="auth-tests", + credentials=[("hello", "iloveyou")], + create_protocol=CustomWebSocketServerProtocol, + ) + + @with_server(create_protocol=create_protocol_custom_protocol) + @with_client(user_info=("hello", "iloveyou")) + def test_basic_auth_custom_protocol(self): + self.assertTrue(CustomWebSocketServerProtocol.used) + del CustomWebSocketServerProtocol.used + self.loop.run_until_complete(self.client.send("Hello!")) + self.loop.run_until_complete(self.client.recv()) + @with_server(create_protocol=create_protocol) def test_basic_auth_missing_credentials(self): with self.assertRaises(InvalidStatusCode) as raised: From cb91aa1575066f6624944cb75bb41d68a45d1b45 Mon Sep 17 00:00:00 2001 From: Janakarajan Natarajan Date: Tue, 18 Aug 2020 22:52:03 +0000 Subject: [PATCH 040/104] Add aarch64 wheel build --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 26e1de60e..e31c9ea0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,13 @@ matrix: python: "3.7" services: - docker + - language: python + dist: xenial + sudo: required + python: "3.7" + arch: arm64 + services: + - docker - os: osx osx_image: xcode8.3 From c39268c4867e41d11c20f7859583761d52a04012 Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Mon, 27 Jul 2020 14:06:08 +0300 Subject: [PATCH 041/104] Fix exception causes in handshake_legacy.py --- src/websockets/handshake_legacy.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/websockets/handshake_legacy.py b/src/websockets/handshake_legacy.py index 7e6acc77d..d34ca5f7f 100644 --- a/src/websockets/handshake_legacy.py +++ b/src/websockets/handshake_legacy.py @@ -91,28 +91,28 @@ def check_request(headers: Headers) -> str: try: s_w_key = headers["Sec-WebSocket-Key"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Key") - except MultipleValuesError: + except KeyError as exc: + raise InvalidHeader("Sec-WebSocket-Key") from exc + except MultipleValuesError as exc: raise InvalidHeader( "Sec-WebSocket-Key", "more than one Sec-WebSocket-Key header found" - ) + ) from exc try: raw_key = base64.b64decode(s_w_key.encode(), validate=True) - except binascii.Error: - raise InvalidHeaderValue("Sec-WebSocket-Key", s_w_key) + except binascii.Error as exc: + raise InvalidHeaderValue("Sec-WebSocket-Key", s_w_key) from exc if len(raw_key) != 16: raise InvalidHeaderValue("Sec-WebSocket-Key", s_w_key) try: s_w_version = headers["Sec-WebSocket-Version"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Version") - except MultipleValuesError: + except KeyError as exc: + raise InvalidHeader("Sec-WebSocket-Version") from exc + except MultipleValuesError as exc: raise InvalidHeader( "Sec-WebSocket-Version", "more than one Sec-WebSocket-Version header found" - ) + ) from exc if s_w_version != "13": raise InvalidHeaderValue("Sec-WebSocket-Version", s_w_version) @@ -168,12 +168,12 @@ def check_response(headers: Headers, key: str) -> None: try: s_w_accept = headers["Sec-WebSocket-Accept"] - except KeyError: - raise InvalidHeader("Sec-WebSocket-Accept") - except MultipleValuesError: + except KeyError as exc: + raise InvalidHeader("Sec-WebSocket-Accept") from exc + except MultipleValuesError as exc: raise InvalidHeader( "Sec-WebSocket-Accept", "more than one Sec-WebSocket-Accept header found" - ) + ) from exc if s_w_accept != accept(key): raise InvalidHeaderValue("Sec-WebSocket-Accept", s_w_accept) From 69cf86724dc2a86f7e57f6393dd322a249dbee17 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 13:29:09 +0100 Subject: [PATCH 042/104] Move question to the FAQ. It was written in the cheatsheet before there was a FAQ. --- docs/cheatsheet.rst | 22 ---------------------- docs/faq.rst | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/docs/cheatsheet.rst b/docs/cheatsheet.rst index f897326a6..4b95c9eea 100644 --- a/docs/cheatsheet.rst +++ b/docs/cheatsheet.rst @@ -85,25 +85,3 @@ in particular. Fortunately Python's official documentation provides advice to .. _develop with asyncio: https://docs.python.org/3/library/asyncio-dev.html -Passing additional arguments to the connection handler ------------------------------------------------------- - -When writing a server, if you need to pass additional arguments to the -connection handler, you can bind them with :func:`functools.partial`:: - - import asyncio - import functools - import websockets - - async def handler(websocket, path, extra_argument): - ... - - bound_handler = functools.partial(handler, extra_argument='spam') - start_server = websockets.serve(bound_handler, '127.0.0.1', 8765) - - asyncio.get_event_loop().run_until_complete(start_server) - asyncio.get_event_loop().run_forever() - -Another way to achieve this result is to define the ``handler`` coroutine in -a scope where the ``extra_argument`` variable exists instead of injecting it -through an argument. diff --git a/docs/faq.rst b/docs/faq.rst index 5e6439055..5748521f0 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -56,6 +56,26 @@ See also Python's documentation about `running blocking code`_. .. _running blocking code: https://docs.python.org/3/library/asyncio-dev.html#running-blocking-code +How can I pass additional arguments to the connection handler? +.............................................................. + +You can bind additional arguments to the connection handler with +:func:`functools.partial`:: + + import asyncio + import functools + import websockets + + async def handler(websocket, path, extra_argument): + ... + + bound_handler = functools.partial(handler, extra_argument='spam') + start_server = websockets.serve(bound_handler, ...) + +Another way to achieve this result is to define the ``handler`` coroutine in +a scope where the ``extra_argument`` variable exists instead of injecting it +through an argument. + How do I get access HTTP headers, for example cookies? ...................................................... From a64136c869c527808c337b13e6dace43ad9d674e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 13:31:38 +0100 Subject: [PATCH 043/104] Remove unfinished sentence. --- docs/faq.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 5748521f0..cd0033734 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -85,8 +85,6 @@ To access HTTP headers during the WebSocket handshake, you can override async def process_request(self, path, request_headers): cookies = request_header["Cookie"] -See - Once the connection is established, they're available in :attr:`~protocol.WebSocketServerProtocol.request_headers`:: From b331e6c9c3d2cfd3d768aa81e396a9e2f977cf88 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 13:37:53 +0100 Subject: [PATCH 044/104] Document how to pass arguments to protocol factory. Fix #851. --- docs/faq.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index cd0033734..4a083e2d0 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -222,6 +222,26 @@ There are several reasons why long-lived connections may be lost: If you're facing a reproducible issue, :ref:`enable debug logs ` to see when and how connections are closed. +How can I pass additional arguments to a custom protocol subclass? +.................................................................. + +You can bind additional arguments to the protocol factory with +:func:`functools.partial`:: + + import asyncio + import functools + import websockets + + class MyServerProtocol(websockets.WebSocketServerProtocol): + def __init__(self, extra_argument, *args, **kwargs): + super().__init__(*args, **kwargs) + # do something with extra_argument + + create_protocol = functools.partial(MyServerProtocol, extra_argument='spam') + start_server = websockets.serve(..., create_protocol=create_protocol) + +This example was for a server. The same pattern applies on a client. + Why do I get the error: ``module 'websockets' has no attribute '...'``? ....................................................................... From 988572074edbde4dce1e49573e9dca05498bb159 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 13:50:45 +0100 Subject: [PATCH 045/104] Brag with # stargazers. Fix #844. --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 064c657bf..0c00b96fb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -111,6 +111,7 @@ 'logo': 'websockets.svg', 'description': 'A library for building WebSocket servers and clients in Python with a focus on correctness and simplicity.', 'github_button': True, + 'github_type': 'star', 'github_user': 'aaugustin', 'github_repo': 'websockets', 'tidelift_url': 'https://tidelift.com/subscription/pkg/pypi-websockets?utm_source=pypi-websockets&utm_medium=referral&utm_campaign=docs', From e6d5da9b94167d875e2fb3936e44665fe0f562bc Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 16:25:33 +0100 Subject: [PATCH 046/104] Include "broadcast" as a search term. Fix #841. --- docs/intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro.rst b/docs/intro.rst index 8be700239..8aaaeddca 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -180,7 +180,7 @@ unregister them when they disconnect. # Register. connected.add(websocket) try: - # Implement logic here. + # Broadcast a message to all connected clients. await asyncio.wait([ws.send("Hello!") for ws in connected]) await asyncio.sleep(10) finally: From f6e03bbd1f0e1affdda16488e46ae488ab0ccfcb Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 17:38:30 +0100 Subject: [PATCH 047/104] Run new version of black. --- src/websockets/client.py | 2 +- src/websockets/connection.py | 5 +- src/websockets/exceptions.py | 5 +- src/websockets/server.py | 3 +- tests/test_auth.py | 3 +- tests/test_connection.py | 200 +++++++++++++++++++++++------------ tests/test_frames.py | 49 +++++++-- tests/test_http11.py | 60 ++++++++--- 8 files changed, 232 insertions(+), 95 deletions(-) diff --git a/src/websockets/client.py b/src/websockets/client.py index a7bfcc4ee..b7e407a45 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -280,7 +280,7 @@ def send_request(self, request: Request) -> None: def parse(self) -> Generator[None, None, None]: response = yield from Response.parse( - self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof, + self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof ) assert self.state == CONNECTING try: diff --git a/src/websockets/connection.py b/src/websockets/connection.py index ac30802db..a98d0b1e7 100644 --- a/src/websockets/connection.py +++ b/src/websockets/connection.py @@ -63,7 +63,10 @@ class State(enum.IntEnum): class Connection: def __init__( - self, side: Side, state: State = OPEN, max_size: Optional[int] = 2 ** 20, + self, + side: Side, + state: State = OPEN, + max_size: Optional[int] = 2 ** 20, ) -> None: # Connection side. CLIENT or SERVER. self.side = side diff --git a/src/websockets/exceptions.py b/src/websockets/exceptions.py index c60a3e10e..84c27692c 100644 --- a/src/websockets/exceptions.py +++ b/src/websockets/exceptions.py @@ -302,7 +302,10 @@ class AbortHandshake(InvalidHandshake): """ def __init__( - self, status: http.HTTPStatus, headers: HeadersLike, body: bytes = b"" + self, + status: http.HTTPStatus, + headers: HeadersLike, + body: bytes = b"", ) -> None: self.status = status self.headers = Headers(headers) diff --git a/src/websockets/server.py b/src/websockets/server.py index 1b03eabee..c2c818ce9 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -242,7 +242,8 @@ def process_origin(self, headers: Headers) -> Optional[Origin]: return origin def process_extensions( - self, headers: Headers, + self, + headers: Headers, ) -> Tuple[Optional[str], List[Extension]]: """ Handle the Sec-WebSocket-Extensions HTTP request header. diff --git a/tests/test_auth.py b/tests/test_auth.py index 68642389e..ce23f913d 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -79,7 +79,8 @@ async def check_credentials(username, password): return password == "iloveyou" create_protocol_check_credentials = basic_auth_protocol_factory( - realm="auth-tests", check_credentials=check_credentials, + realm="auth-tests", + check_credentials=check_credentials, ) @with_server(create_protocol=create_protocol_check_credentials) diff --git a/tests/test_connection.py b/tests/test_connection.py index 5c0f7302f..d47147d64 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -59,7 +59,9 @@ def assertConnectionClosing(self, connection, code=None, reason=""): """ close_frame = Frame( - True, OP_CLOSE, b"" if code is None else serialize_close(code, reason), + True, + OP_CLOSE, + b"" if code is None else serialize_close(code, reason), ) # A close frame was received. self.assertFrameReceived(connection, close_frame) @@ -74,7 +76,9 @@ def assertConnectionFailing(self, connection, code=None, reason=""): """ close_frame = Frame( - True, OP_CLOSE, b"" if code is None else serialize_close(code, reason), + True, + OP_CLOSE, + b"" if code is None else serialize_close(code, reason), ) # No frame was received. self.assertFrameReceived(connection, None) @@ -108,14 +112,16 @@ def test_client_receives_unmasked_frame(self): client = Connection(Side.CLIENT) client.receive_data(self.unmasked_text_frame_date) self.assertFrameReceived( - client, Frame(True, OP_TEXT, b"Spam"), + client, + Frame(True, OP_TEXT, b"Spam"), ) def test_server_receives_masked_frame(self): server = Connection(Side.SERVER) server.receive_data(self.masked_text_frame_data) self.assertFrameReceived( - server, Frame(True, OP_TEXT, b"Spam"), + server, + Frame(True, OP_TEXT, b"Spam"), ) def test_client_receives_masked_frame(self): @@ -228,14 +234,16 @@ def test_client_receives_text(self): client = Connection(Side.CLIENT) client.receive_data(b"\x81\x04\xf0\x9f\x98\x80") self.assertFrameReceived( - client, Frame(True, OP_TEXT, "😀".encode()), + client, + Frame(True, OP_TEXT, "😀".encode()), ) def test_server_receives_text(self): server = Connection(Side.SERVER) server.receive_data(b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80") self.assertFrameReceived( - server, Frame(True, OP_TEXT, "😀".encode()), + server, + Frame(True, OP_TEXT, "😀".encode()), ) def test_client_receives_text_over_size_limit(self): @@ -256,14 +264,16 @@ def test_client_receives_text_without_size_limit(self): client = Connection(Side.CLIENT, max_size=None) client.receive_data(b"\x81\x04\xf0\x9f\x98\x80") self.assertFrameReceived( - client, Frame(True, OP_TEXT, "😀".encode()), + client, + Frame(True, OP_TEXT, "😀".encode()), ) def test_server_receives_text_without_size_limit(self): server = Connection(Side.SERVER, max_size=None) server.receive_data(b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80") self.assertFrameReceived( - server, Frame(True, OP_TEXT, "😀".encode()), + server, + Frame(True, OP_TEXT, "😀".encode()), ) def test_client_sends_fragmented_text(self): @@ -293,37 +303,44 @@ def test_client_receives_fragmented_text(self): client = Connection(Side.CLIENT) client.receive_data(b"\x01\x02\xf0\x9f") self.assertFrameReceived( - client, Frame(False, OP_TEXT, "😀".encode()[:2]), + client, + Frame(False, OP_TEXT, "😀".encode()[:2]), ) client.receive_data(b"\x00\x04\x98\x80\xf0\x9f") self.assertFrameReceived( - client, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + client, + Frame(False, OP_CONT, "😀😀".encode()[2:6]), ) client.receive_data(b"\x80\x02\x98\x80") self.assertFrameReceived( - client, Frame(True, OP_CONT, "😀".encode()[2:]), + client, + Frame(True, OP_CONT, "😀".encode()[2:]), ) def test_server_receives_fragmented_text(self): server = Connection(Side.SERVER) server.receive_data(b"\x01\x82\x00\x00\x00\x00\xf0\x9f") self.assertFrameReceived( - server, Frame(False, OP_TEXT, "😀".encode()[:2]), + server, + Frame(False, OP_TEXT, "😀".encode()[:2]), ) server.receive_data(b"\x00\x84\x00\x00\x00\x00\x98\x80\xf0\x9f") self.assertFrameReceived( - server, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + server, + Frame(False, OP_CONT, "😀😀".encode()[2:6]), ) server.receive_data(b"\x80\x82\x00\x00\x00\x00\x98\x80") self.assertFrameReceived( - server, Frame(True, OP_CONT, "😀".encode()[2:]), + server, + Frame(True, OP_CONT, "😀".encode()[2:]), ) def test_client_receives_fragmented_text_over_size_limit(self): client = Connection(Side.CLIENT, max_size=3) client.receive_data(b"\x01\x02\xf0\x9f") self.assertFrameReceived( - client, Frame(False, OP_TEXT, "😀".encode()[:2]), + client, + Frame(False, OP_TEXT, "😀".encode()[:2]), ) with self.assertRaises(PayloadTooBig) as raised: client.receive_data(b"\x80\x02\x98\x80") @@ -334,7 +351,8 @@ def test_server_receives_fragmented_text_over_size_limit(self): server = Connection(Side.SERVER, max_size=3) server.receive_data(b"\x01\x82\x00\x00\x00\x00\xf0\x9f") self.assertFrameReceived( - server, Frame(False, OP_TEXT, "😀".encode()[:2]), + server, + Frame(False, OP_TEXT, "😀".encode()[:2]), ) with self.assertRaises(PayloadTooBig) as raised: server.receive_data(b"\x80\x82\x00\x00\x00\x00\x98\x80") @@ -345,30 +363,36 @@ def test_client_receives_fragmented_text_without_size_limit(self): client = Connection(Side.CLIENT, max_size=None) client.receive_data(b"\x01\x02\xf0\x9f") self.assertFrameReceived( - client, Frame(False, OP_TEXT, "😀".encode()[:2]), + client, + Frame(False, OP_TEXT, "😀".encode()[:2]), ) client.receive_data(b"\x00\x04\x98\x80\xf0\x9f") self.assertFrameReceived( - client, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + client, + Frame(False, OP_CONT, "😀😀".encode()[2:6]), ) client.receive_data(b"\x80\x02\x98\x80") self.assertFrameReceived( - client, Frame(True, OP_CONT, "😀".encode()[2:]), + client, + Frame(True, OP_CONT, "😀".encode()[2:]), ) def test_server_receives_fragmented_text_without_size_limit(self): server = Connection(Side.SERVER, max_size=None) server.receive_data(b"\x01\x82\x00\x00\x00\x00\xf0\x9f") self.assertFrameReceived( - server, Frame(False, OP_TEXT, "😀".encode()[:2]), + server, + Frame(False, OP_TEXT, "😀".encode()[:2]), ) server.receive_data(b"\x00\x84\x00\x00\x00\x00\x98\x80\xf0\x9f") self.assertFrameReceived( - server, Frame(False, OP_CONT, "😀😀".encode()[2:6]), + server, + Frame(False, OP_CONT, "😀😀".encode()[2:6]), ) server.receive_data(b"\x80\x82\x00\x00\x00\x00\x98\x80") self.assertFrameReceived( - server, Frame(True, OP_CONT, "😀".encode()[2:]), + server, + Frame(True, OP_CONT, "😀".encode()[2:]), ) def test_client_sends_unexpected_text(self): @@ -389,7 +413,8 @@ def test_client_receives_unexpected_text(self): client = Connection(Side.CLIENT) client.receive_data(b"\x01\x00") self.assertFrameReceived( - client, Frame(False, OP_TEXT, b""), + client, + Frame(False, OP_TEXT, b""), ) with self.assertRaises(ProtocolError) as raised: client.receive_data(b"\x01\x00") @@ -400,7 +425,8 @@ def test_server_receives_unexpected_text(self): server = Connection(Side.SERVER) server.receive_data(b"\x01\x80\x00\x00\x00\x00") self.assertFrameReceived( - server, Frame(False, OP_TEXT, b""), + server, + Frame(False, OP_TEXT, b""), ) with self.assertRaises(ProtocolError) as raised: server.receive_data(b"\x01\x80\x00\x00\x00\x00") @@ -462,14 +488,16 @@ def test_client_receives_binary(self): client = Connection(Side.CLIENT) client.receive_data(b"\x82\x04\x01\x02\xfe\xff") self.assertFrameReceived( - client, Frame(True, OP_BINARY, b"\x01\x02\xfe\xff"), + client, + Frame(True, OP_BINARY, b"\x01\x02\xfe\xff"), ) def test_server_receives_binary(self): server = Connection(Side.SERVER) server.receive_data(b"\x82\x84\x00\x00\x00\x00\x01\x02\xfe\xff") self.assertFrameReceived( - server, Frame(True, OP_BINARY, b"\x01\x02\xfe\xff"), + server, + Frame(True, OP_BINARY, b"\x01\x02\xfe\xff"), ) def test_client_receives_binary_over_size_limit(self): @@ -513,37 +541,44 @@ def test_client_receives_fragmented_binary(self): client = Connection(Side.CLIENT) client.receive_data(b"\x02\x02\x01\x02") self.assertFrameReceived( - client, Frame(False, OP_BINARY, b"\x01\x02"), + client, + Frame(False, OP_BINARY, b"\x01\x02"), ) client.receive_data(b"\x00\x04\xfe\xff\x01\x02") self.assertFrameReceived( - client, Frame(False, OP_CONT, b"\xfe\xff\x01\x02"), + client, + Frame(False, OP_CONT, b"\xfe\xff\x01\x02"), ) client.receive_data(b"\x80\x02\xfe\xff") self.assertFrameReceived( - client, Frame(True, OP_CONT, b"\xfe\xff"), + client, + Frame(True, OP_CONT, b"\xfe\xff"), ) def test_server_receives_fragmented_binary(self): server = Connection(Side.SERVER) server.receive_data(b"\x02\x82\x00\x00\x00\x00\x01\x02") self.assertFrameReceived( - server, Frame(False, OP_BINARY, b"\x01\x02"), + server, + Frame(False, OP_BINARY, b"\x01\x02"), ) server.receive_data(b"\x00\x84\x00\x00\x00\x00\xee\xff\x01\x02") self.assertFrameReceived( - server, Frame(False, OP_CONT, b"\xee\xff\x01\x02"), + server, + Frame(False, OP_CONT, b"\xee\xff\x01\x02"), ) server.receive_data(b"\x80\x82\x00\x00\x00\x00\xfe\xff") self.assertFrameReceived( - server, Frame(True, OP_CONT, b"\xfe\xff"), + server, + Frame(True, OP_CONT, b"\xfe\xff"), ) def test_client_receives_fragmented_binary_over_size_limit(self): client = Connection(Side.CLIENT, max_size=3) client.receive_data(b"\x02\x02\x01\x02") self.assertFrameReceived( - client, Frame(False, OP_BINARY, b"\x01\x02"), + client, + Frame(False, OP_BINARY, b"\x01\x02"), ) with self.assertRaises(PayloadTooBig) as raised: client.receive_data(b"\x80\x02\xfe\xff") @@ -554,7 +589,8 @@ def test_server_receives_fragmented_binary_over_size_limit(self): server = Connection(Side.SERVER, max_size=3) server.receive_data(b"\x02\x82\x00\x00\x00\x00\x01\x02") self.assertFrameReceived( - server, Frame(False, OP_BINARY, b"\x01\x02"), + server, + Frame(False, OP_BINARY, b"\x01\x02"), ) with self.assertRaises(PayloadTooBig) as raised: server.receive_data(b"\x80\x82\x00\x00\x00\x00\xfe\xff") @@ -579,7 +615,8 @@ def test_client_receives_unexpected_binary(self): client = Connection(Side.CLIENT) client.receive_data(b"\x02\x00") self.assertFrameReceived( - client, Frame(False, OP_BINARY, b""), + client, + Frame(False, OP_BINARY, b""), ) with self.assertRaises(ProtocolError) as raised: client.receive_data(b"\x02\x00") @@ -590,7 +627,8 @@ def test_server_receives_unexpected_binary(self): server = Connection(Side.SERVER) server.receive_data(b"\x02\x80\x00\x00\x00\x00") self.assertFrameReceived( - server, Frame(False, OP_BINARY, b""), + server, + Frame(False, OP_BINARY, b""), ) with self.assertRaises(ProtocolError) as raised: server.receive_data(b"\x02\x80\x00\x00\x00\x00") @@ -843,20 +881,24 @@ def test_client_receives_ping(self): client = Connection(Side.CLIENT) client.receive_data(b"\x89\x00") self.assertFrameReceived( - client, Frame(True, OP_PING, b""), + client, + Frame(True, OP_PING, b""), ) self.assertFrameSent( - client, Frame(True, OP_PONG, b""), + client, + Frame(True, OP_PONG, b""), ) def test_server_receives_ping(self): server = Connection(Side.SERVER) server.receive_data(b"\x89\x80\x00\x44\x88\xcc") self.assertFrameReceived( - server, Frame(True, OP_PING, b""), + server, + Frame(True, OP_PING, b""), ) self.assertFrameSent( - server, Frame(True, OP_PONG, b""), + server, + Frame(True, OP_PONG, b""), ) def test_client_sends_ping_with_data(self): @@ -876,20 +918,24 @@ def test_client_receives_ping_with_data(self): client = Connection(Side.CLIENT) client.receive_data(b"\x89\x04\x22\x66\xaa\xee") self.assertFrameReceived( - client, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + client, + Frame(True, OP_PING, b"\x22\x66\xaa\xee"), ) self.assertFrameSent( - client, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + client, + Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), ) def test_server_receives_ping_with_data(self): server = Connection(Side.SERVER) server.receive_data(b"\x89\x84\x00\x44\x88\xcc\x22\x22\x22\x22") self.assertFrameReceived( - server, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + server, + Frame(True, OP_PING, b"\x22\x66\xaa\xee"), ) self.assertFrameSent( - server, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + server, + Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), ) def test_client_sends_fragmented_ping_frame(self): @@ -953,7 +999,8 @@ def test_client_receives_ping_after_receiving_close(self): self.assertConnectionClosing(client, 1000) client.receive_data(b"\x89\x04\x22\x66\xaa\xee") self.assertFrameReceived( - client, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + client, + Frame(True, OP_PING, b"\x22\x66\xaa\xee"), ) self.assertFrameSent(client, None) @@ -963,7 +1010,8 @@ def test_server_receives_ping_after_receiving_close(self): self.assertConnectionClosing(server, 1001) server.receive_data(b"\x89\x84\x00\x44\x88\xcc\x22\x22\x22\x22") self.assertFrameReceived( - server, Frame(True, OP_PING, b"\x22\x66\xaa\xee"), + server, + Frame(True, OP_PING, b"\x22\x66\xaa\xee"), ) self.assertFrameSent(server, None) @@ -989,14 +1037,16 @@ def test_client_receives_pong(self): client = Connection(Side.CLIENT) client.receive_data(b"\x8a\x00") self.assertFrameReceived( - client, Frame(True, OP_PONG, b""), + client, + Frame(True, OP_PONG, b""), ) def test_server_receives_pong(self): server = Connection(Side.SERVER) server.receive_data(b"\x8a\x80\x00\x44\x88\xcc") self.assertFrameReceived( - server, Frame(True, OP_PONG, b""), + server, + Frame(True, OP_PONG, b""), ) def test_client_sends_pong_with_data(self): @@ -1016,14 +1066,16 @@ def test_client_receives_pong_with_data(self): client = Connection(Side.CLIENT) client.receive_data(b"\x8a\x04\x22\x66\xaa\xee") self.assertFrameReceived( - client, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + client, + Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), ) def test_server_receives_pong_with_data(self): server = Connection(Side.SERVER) server.receive_data(b"\x8a\x84\x00\x44\x88\xcc\x22\x22\x22\x22") self.assertFrameReceived( - server, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + server, + Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), ) def test_client_sends_fragmented_pong_frame(self): @@ -1077,7 +1129,8 @@ def test_client_receives_pong_after_receiving_close(self): self.assertConnectionClosing(client, 1000) client.receive_data(b"\x8a\x04\x22\x66\xaa\xee") self.assertFrameReceived( - client, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + client, + Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), ) def test_server_receives_pong_after_receiving_close(self): @@ -1086,7 +1139,8 @@ def test_server_receives_pong_after_receiving_close(self): self.assertConnectionClosing(server, 1001) server.receive_data(b"\x8a\x84\x00\x44\x88\xcc\x22\x22\x22\x22") self.assertFrameReceived( - server, Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), + server, + Frame(True, OP_PONG, b"\x22\x66\xaa\xee"), ) @@ -1128,52 +1182,64 @@ def test_client_receive_ping_pong_in_fragmented_message(self): client = Connection(Side.CLIENT) client.receive_data(b"\x01\x04Spam") self.assertFrameReceived( - client, Frame(False, OP_TEXT, b"Spam"), + client, + Frame(False, OP_TEXT, b"Spam"), ) client.receive_data(b"\x89\x04Ping") self.assertFrameReceived( - client, Frame(True, OP_PING, b"Ping"), + client, + Frame(True, OP_PING, b"Ping"), ) self.assertFrameSent( - client, Frame(True, OP_PONG, b"Ping"), + client, + Frame(True, OP_PONG, b"Ping"), ) client.receive_data(b"\x00\x03Ham") self.assertFrameReceived( - client, Frame(False, OP_CONT, b"Ham"), + client, + Frame(False, OP_CONT, b"Ham"), ) client.receive_data(b"\x8a\x04Pong") self.assertFrameReceived( - client, Frame(True, OP_PONG, b"Pong"), + client, + Frame(True, OP_PONG, b"Pong"), ) client.receive_data(b"\x80\x04Eggs") self.assertFrameReceived( - client, Frame(True, OP_CONT, b"Eggs"), + client, + Frame(True, OP_CONT, b"Eggs"), ) def test_server_receive_ping_pong_in_fragmented_message(self): server = Connection(Side.SERVER) server.receive_data(b"\x01\x84\x00\x00\x00\x00Spam") self.assertFrameReceived( - server, Frame(False, OP_TEXT, b"Spam"), + server, + Frame(False, OP_TEXT, b"Spam"), ) server.receive_data(b"\x89\x84\x00\x00\x00\x00Ping") self.assertFrameReceived( - server, Frame(True, OP_PING, b"Ping"), + server, + Frame(True, OP_PING, b"Ping"), ) self.assertFrameSent( - server, Frame(True, OP_PONG, b"Ping"), + server, + Frame(True, OP_PONG, b"Ping"), ) server.receive_data(b"\x00\x83\x00\x00\x00\x00Ham") self.assertFrameReceived( - server, Frame(False, OP_CONT, b"Ham"), + server, + Frame(False, OP_CONT, b"Ham"), ) server.receive_data(b"\x8a\x84\x00\x00\x00\x00Pong") self.assertFrameReceived( - server, Frame(True, OP_PONG, b"Pong"), + server, + Frame(True, OP_PONG, b"Pong"), ) server.receive_data(b"\x80\x84\x00\x00\x00\x00Eggs") self.assertFrameReceived( - server, Frame(True, OP_CONT, b"Eggs"), + server, + Frame(True, OP_CONT, b"Eggs"), ) def test_client_send_close_in_fragmented_message(self): @@ -1205,7 +1271,8 @@ def test_client_receive_close_in_fragmented_message(self): client = Connection(Side.CLIENT) client.receive_data(b"\x01\x04Spam") self.assertFrameReceived( - client, Frame(False, OP_TEXT, b"Spam"), + client, + Frame(False, OP_TEXT, b"Spam"), ) # The spec says: "An endpoint MUST be capable of handling control # frames in the middle of a fragmented message." However, since the @@ -1220,7 +1287,8 @@ def test_server_receive_close_in_fragmented_message(self): server = Connection(Side.SERVER) server.receive_data(b"\x01\x84\x00\x00\x00\x00Spam") self.assertFrameReceived( - server, Frame(False, OP_TEXT, b"Spam"), + server, + Frame(False, OP_TEXT, b"Spam"), ) # The spec says: "An endpoint MUST be capable of handling control # frames in the middle of a fragmented message." However, since the diff --git a/tests/test_frames.py b/tests/test_frames.py index 514fe7c54..4d10c6ef2 100644 --- a/tests/test_frames.py +++ b/tests/test_frames.py @@ -22,7 +22,7 @@ def parse(self, data, mask, max_size=None, extensions=None): reader.feed_data(data) reader.feed_eof() parser = Frame.parse( - reader.read_exact, mask=mask, max_size=max_size, extensions=extensions, + reader.read_exact, mask=mask, max_size=max_size, extensions=extensions ) return self.assertGeneratorReturns(parser) @@ -47,7 +47,9 @@ def assertFrameData(self, frame, data, mask, extensions=None): class FrameTests(FramesTestCase): def test_text_unmasked(self): self.assertFrameData( - Frame(True, OP_TEXT, b"Spam"), b"\x81\x04Spam", mask=False, + Frame(True, OP_TEXT, b"Spam"), + b"\x81\x04Spam", + mask=False, ) def test_text_masked(self): @@ -59,7 +61,9 @@ def test_text_masked(self): def test_binary_unmasked(self): self.assertFrameData( - Frame(True, OP_BINARY, b"Eggs"), b"\x82\x04Eggs", mask=False, + Frame(True, OP_BINARY, b"Eggs"), + b"\x82\x04Eggs", + mask=False, ) def test_binary_masked(self): @@ -84,13 +88,25 @@ def test_non_ascii_text_masked(self): ) def test_close(self): - self.assertFrameData(Frame(True, OP_CLOSE, b""), b"\x88\x00", mask=False) + self.assertFrameData( + Frame(True, OP_CLOSE, b""), + b"\x88\x00", + mask=False, + ) def test_ping(self): - self.assertFrameData(Frame(True, OP_PING, b"ping"), b"\x89\x04ping", mask=False) + self.assertFrameData( + Frame(True, OP_PING, b"ping"), + b"\x89\x04ping", + mask=False, + ) def test_pong(self): - self.assertFrameData(Frame(True, OP_PONG, b"pong"), b"\x8a\x04pong", mask=False) + self.assertFrameData( + Frame(True, OP_PONG, b"pong"), + b"\x8a\x04pong", + mask=False, + ) def test_long(self): self.assertFrameData( @@ -179,23 +195,34 @@ def decode(frame, *, max_size=None): class PrepareDataTests(unittest.TestCase): def test_prepare_data_str(self): - self.assertEqual(prepare_data("café"), (OP_TEXT, b"caf\xc3\xa9")) + self.assertEqual( + prepare_data("café"), + (OP_TEXT, b"caf\xc3\xa9"), + ) def test_prepare_data_bytes(self): - self.assertEqual(prepare_data(b"tea"), (OP_BINARY, b"tea")) + self.assertEqual( + prepare_data(b"tea"), + (OP_BINARY, b"tea"), + ) def test_prepare_data_bytearray(self): self.assertEqual( - prepare_data(bytearray(b"tea")), (OP_BINARY, bytearray(b"tea")) + prepare_data(bytearray(b"tea")), + (OP_BINARY, bytearray(b"tea")), ) def test_prepare_data_memoryview(self): self.assertEqual( - prepare_data(memoryview(b"tea")), (OP_BINARY, memoryview(b"tea")) + prepare_data(memoryview(b"tea")), + (OP_BINARY, memoryview(b"tea")), ) def test_prepare_data_non_contiguous_memoryview(self): - self.assertEqual(prepare_data(memoryview(b"tteeaa")[::2]), (OP_BINARY, b"tea")) + self.assertEqual( + prepare_data(memoryview(b"tteeaa")[::2]), + (OP_BINARY, b"tea"), + ) def test_prepare_data_list(self): with self.assertRaises(TypeError): diff --git a/tests/test_http11.py b/tests/test_http11.py index 9e4d70620..1cca2053f 100644 --- a/tests/test_http11.py +++ b/tests/test_http11.py @@ -37,32 +37,45 @@ def test_parse_empty(self): with self.assertRaises(EOFError) as raised: next(self.parse()) self.assertEqual( - str(raised.exception), "connection closed while reading HTTP request line" + str(raised.exception), + "connection closed while reading HTTP request line", ) def test_parse_invalid_request_line(self): self.reader.feed_data(b"GET /\r\n\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "invalid HTTP request line: GET /") + self.assertEqual( + str(raised.exception), + "invalid HTTP request line: GET /", + ) def test_parse_unsupported_method(self): self.reader.feed_data(b"OPTIONS * HTTP/1.1\r\n\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "unsupported HTTP method: OPTIONS") + self.assertEqual( + str(raised.exception), + "unsupported HTTP method: OPTIONS", + ) def test_parse_unsupported_version(self): self.reader.feed_data(b"GET /chat HTTP/1.0\r\n\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "unsupported HTTP version: HTTP/1.0") + self.assertEqual( + str(raised.exception), + "unsupported HTTP version: HTTP/1.0", + ) def test_parse_invalid_header(self): self.reader.feed_data(b"GET /chat HTTP/1.1\r\nOops\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "invalid HTTP header line: Oops") + self.assertEqual( + str(raised.exception), + "invalid HTTP header line: Oops", + ) def test_serialize(self): # Example from the protocol overview in RFC 6455 @@ -101,7 +114,7 @@ def setUp(self): def parse(self): return Response.parse( - self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof, + self.reader.read_line, self.reader.read_exact, self.reader.read_to_eof ) def test_parse(self): @@ -132,37 +145,55 @@ def test_parse_invalid_status_line(self): self.reader.feed_data(b"Hello!\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "invalid HTTP status line: Hello!") + self.assertEqual( + str(raised.exception), + "invalid HTTP status line: Hello!", + ) def test_parse_unsupported_version(self): self.reader.feed_data(b"HTTP/1.0 400 Bad Request\r\n\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "unsupported HTTP version: HTTP/1.0") + self.assertEqual( + str(raised.exception), + "unsupported HTTP version: HTTP/1.0", + ) def test_parse_invalid_status(self): self.reader.feed_data(b"HTTP/1.1 OMG WTF\r\n\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "invalid HTTP status code: OMG") + self.assertEqual( + str(raised.exception), + "invalid HTTP status code: OMG", + ) def test_parse_unsupported_status(self): self.reader.feed_data(b"HTTP/1.1 007 My name is Bond\r\n\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "unsupported HTTP status code: 007") + self.assertEqual( + str(raised.exception), + "unsupported HTTP status code: 007", + ) def test_parse_invalid_reason(self): self.reader.feed_data(b"HTTP/1.1 200 \x7f\r\n\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "invalid HTTP reason phrase: \x7f") + self.assertEqual( + str(raised.exception), + "invalid HTTP reason phrase: \x7f", + ) def test_parse_invalid_header(self): self.reader.feed_data(b"HTTP/1.1 500 Internal Server Error\r\nOops\r\n") with self.assertRaises(ValueError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "invalid HTTP header line: Oops") + self.assertEqual( + str(raised.exception), + "invalid HTTP header line: Oops", + ) def test_parse_body_with_content_length(self): self.reader.feed_data( @@ -183,7 +214,10 @@ def test_parse_body_with_transfer_encoding(self): self.reader.feed_data(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") with self.assertRaises(NotImplementedError) as raised: next(self.parse()) - self.assertEqual(str(raised.exception), "transfer codings aren't supported") + self.assertEqual( + str(raised.exception), + "transfer codings aren't supported", + ) def test_parse_body_no_content(self): self.reader.feed_data(b"HTTP/1.1 204 No Content\r\n\r\n") From 5bce4c1c5e59c8c3f5ec45de1c94f9047126b885 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 17:44:11 +0100 Subject: [PATCH 048/104] Support IRIs in addition to URIs. Fix #832. --- docs/changelog.rst | 2 ++ src/websockets/uri.py | 18 ++++++++++++++++++ tests/test_uri.py | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 68ec6f80c..4c0eb7d2c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ Changelog Aliases provide backwards compatibility for all previously public APIs. +* Added support for IRIs in addition to URIs. + 8.1 ... diff --git a/src/websockets/uri.py b/src/websockets/uri.py index 6669e5668..ce21b445b 100644 --- a/src/websockets/uri.py +++ b/src/websockets/uri.py @@ -49,6 +49,10 @@ class WebSocketURI(NamedTuple): WebSocketURI.user_info.__doc__ = "" +# All characters from the gen-delims and sub-delims sets in RFC 3987. +DELIMS = ":/?#[]@!$&'()*+,;=" + + def parse_uri(uri: str) -> WebSocketURI: """ Parse and validate a WebSocket URI. @@ -78,4 +82,18 @@ def parse_uri(uri: str) -> WebSocketURI: if parsed.password is None: raise InvalidURI(uri) user_info = (parsed.username, parsed.password) + + try: + uri.encode("ascii") + except UnicodeEncodeError: + # Input contains non-ASCII characters. + # It must be an IRI. Convert it to a URI. + host = host.encode("idna").decode() + resource_name = urllib.parse.quote(resource_name, safe=DELIMS) + if user_info is not None: + user_info = ( + urllib.parse.quote(user_info[0], safe=DELIMS), + urllib.parse.quote(user_info[1], safe=DELIMS), + ) + return WebSocketURI(secure, host, port, resource_name, user_info) diff --git a/tests/test_uri.py b/tests/test_uri.py index e41860b8e..9eeb8431d 100644 --- a/tests/test_uri.py +++ b/tests/test_uri.py @@ -10,6 +10,11 @@ ("ws://localhost/path?query", (False, "localhost", 80, "/path?query", None)), ("WS://LOCALHOST/PATH?QUERY", (False, "localhost", 80, "/PATH?QUERY", None)), ("ws://user:pass@localhost/", (False, "localhost", 80, "/", ("user", "pass"))), + ("ws://høst/", (False, "xn--hst-0na", 80, "/", None)), + ( + "ws://üser:påss@høst/πass", + (False, "xn--hst-0na", 80, "/%CF%80ass", ("%C3%BCser", "p%C3%A5ss")), + ), ] INVALID_URIS = [ From 72d32619650eace78a4d7e797de9369fbee10ada Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 17:54:07 +0100 Subject: [PATCH 049/104] Improve detection of broken connections. Refs #810. --- src/websockets/protocol.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 92ce8e305..39b578aba 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -877,10 +877,11 @@ async def transfer_data(self) -> None: self.transfer_data_exc = exc self.fail_connection(1002) - except (ConnectionError, EOFError) as exc: + except (ConnectionError, TimeoutError, EOFError) as exc: # Reading data with self.reader.readexactly may raise: # - most subclasses of ConnectionError if the TCP connection # breaks, is reset, or is aborted; + # - TimeoutError if the TCP connection times out; # - IncompleteReadError, a subclass of EOFError, if fewer # bytes are available than requested. self.transfer_data_exc = exc From 8061b03b803fb1ce2c7dfcf7bf3cd48f41d34b83 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 18:18:54 +0100 Subject: [PATCH 050/104] Remove loop argument to asyncio.Queue. Prepare compatibility with Python 3.10. Fix #801. --- src/websockets/__main__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/websockets/__main__.py b/src/websockets/__main__.py index 1a720498d..5013ca04f 100644 --- a/src/websockets/__main__.py +++ b/src/websockets/__main__.py @@ -176,8 +176,13 @@ def main() -> None: # Create an event loop that will run in a background thread. loop = asyncio.new_event_loop() + # Due to zealous removal of the loop parameter in the Queue constructor, + # we need a factory coroutine to run in the freshly created event loop. + async def queue_factory() -> asyncio.Queue[str]: + return asyncio.Queue() + # Create a queue of user inputs. There's no need to limit its size. - inputs: asyncio.Queue[str] = asyncio.Queue(loop=loop) + inputs: asyncio.Queue[str] = loop.run_until_complete(queue_factory()) # Create a stop condition when receiving SIGINT or SIGTERM. stop: asyncio.Future[None] = loop.create_future() From 867a00e5bafa1c8ad412eef06a5b09bac40694dc Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 18:32:38 +0100 Subject: [PATCH 051/104] Eliminate ResourceWarning. --- src/websockets/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/websockets/__main__.py b/src/websockets/__main__.py index 5013ca04f..bce3e4bbb 100644 --- a/src/websockets/__main__.py +++ b/src/websockets/__main__.py @@ -206,6 +206,10 @@ async def queue_factory() -> asyncio.Queue[str]: # Wait for the event loop to terminate. thread.join() + # For reasons unclear, even though the loop is closed in the thread, + # it still thinks it's running here. + loop.close() + if __name__ == "__main__": main() From 32c9036ac5eee02e5167f93474b22e9cddbc78bd Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 18:33:03 +0100 Subject: [PATCH 052/104] Mask expected deprecation warning. --- src/websockets/protocol.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 39b578aba..677d50f2c 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -54,7 +54,14 @@ prepare_data, serialize_close, ) -from .framing import Frame + + +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", "websockets.framing is deprecated", DeprecationWarning + ) + from .framing import Frame + from .typing import Data, Subprotocol From 07775cfaa07b2fb2e31622af03a4fa62820482fb Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 18:56:03 +0100 Subject: [PATCH 053/104] Mark code for removal. Refs #803. --- src/websockets/asyncio_client.py | 2 ++ src/websockets/asyncio_server.py | 2 ++ src/websockets/protocol.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/websockets/asyncio_client.py b/src/websockets/asyncio_client.py index efa29b69a..43e3c1cd2 100644 --- a/src/websockets/asyncio_client.py +++ b/src/websockets/asyncio_client.py @@ -101,6 +101,8 @@ async def read_http_response(self) -> Tuple[int, Headers]: """ try: status_code, reason, headers = await read_response(self.reader) + # Remove this branch when dropping support for Python < 3.8 + # because CancelledError no longer inherits Exception. except asyncio.CancelledError: # pragma: no cover raise except Exception as exc: diff --git a/src/websockets/asyncio_server.py b/src/websockets/asyncio_server.py index fe61c7ddc..b4f7fbc92 100644 --- a/src/websockets/asyncio_server.py +++ b/src/websockets/asyncio_server.py @@ -135,6 +135,8 @@ async def handler(self) -> None: available_subprotocols=self.available_subprotocols, extra_headers=self.extra_headers, ) + # Remove this branch when dropping support for Python < 3.8 + # because CancelledError no longer inherits Exception. except asyncio.CancelledError: # pragma: no cover raise except ConnectionError: diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 677d50f2c..ba4fc1d3c 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -1169,6 +1169,8 @@ async def keepalive_ping(self) -> None: self.fail_connection(1011) break + # Remove this branch when dropping support for Python < 3.8 + # because CancelledError no longer inherits Exception. except asyncio.CancelledError: raise From a58540d681fc858fc43fcfaf7a6be33f177446a7 Mon Sep 17 00:00:00 2001 From: konichuvak Date: Thu, 27 Aug 2020 16:26:46 -0400 Subject: [PATCH 054/104] Adds 1012-1014 close codes. Also replac. `list` with a `set` for faster close code lookups. --- src/websockets/exceptions.py | 4 ++++ src/websockets/frames.py | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/websockets/exceptions.py b/src/websockets/exceptions.py index 84c27692c..bdadae05e 100644 --- a/src/websockets/exceptions.py +++ b/src/websockets/exceptions.py @@ -68,6 +68,7 @@ class WebSocketException(Exception): """ +# See https://www.iana.org/assignments/websocket/websocket.xhtml CLOSE_CODES = { 1000: "OK", 1001: "going away", @@ -81,6 +82,9 @@ class WebSocketException(Exception): 1009: "message too big", 1010: "extension required", 1011: "unexpected error", + 1012: "service restart", + 1013: "try again later", + 1014: "bad gateway", 1015: "TLS failure [internal]", } diff --git a/src/websockets/frames.py b/src/websockets/frames.py index 2ff9dbd91..74223c0e8 100644 --- a/src/websockets/frames.py +++ b/src/websockets/frames.py @@ -53,8 +53,21 @@ class Opcode(enum.IntEnum): CTRL_OPCODES = OP_CLOSE, OP_PING, OP_PONG # Close code that are allowed in a close frame. -# Using a list optimizes `code in EXTERNAL_CLOSE_CODES`. -EXTERNAL_CLOSE_CODES = [1000, 1001, 1002, 1003, 1007, 1008, 1009, 1010, 1011] +# Using a set optimizes `code in EXTERNAL_CLOSE_CODES`. +EXTERNAL_CLOSE_CODES = { + 1000, + 1001, + 1002, + 1003, + 1007, + 1008, + 1009, + 1010, + 1011, + 1012, + 1013, + 1014, +} # Consider converting to a dataclass when dropping support for Python < 3.7. From 189671d990a3ecf2d8bf5c7e0c4d97abc9167c20 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 19:06:18 +0100 Subject: [PATCH 055/104] Add changelog for previous commit. --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c0eb7d2c..c131f0528 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -31,6 +31,8 @@ Changelog * Added compatibility with Python 3.8. +* Added close codes 1012, 1013, and 1014. + 8.0.2 ..... From b39f62a066bde151b7551a0d445705481e247e9b Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Nov 2020 20:19:47 +0100 Subject: [PATCH 056/104] Log exceptions consistently. This was the only use of the exception method (vs. exc_info=True). --- src/websockets/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/websockets/connection.py b/src/websockets/connection.py index a98d0b1e7..4a75bede9 100644 --- a/src/websockets/connection.py +++ b/src/websockets/connection.py @@ -282,7 +282,7 @@ def step_parser(self) -> None: self.parser_exc = exc raise except Exception as exc: - logger.exception("unexpected exception in parser") + logger.error("unexpected exception in parser", exc_info=True) # Don't include exception details, which may be security-sensitive. self.fail_connection(1011) self.parser_exc = exc From 984da0efa69c0fe3518f1bb81d43775f5ef66902 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 28 Nov 2020 13:21:31 +0100 Subject: [PATCH 057/104] Rename bytes_to_send to data_to_send. Since this function doesn't return bytes, but an iterable of bytes, the name was confusing. --- src/websockets/connection.py | 6 +-- tests/test_client.py | 2 +- tests/test_connection.py | 94 ++++++++++++++++++------------------ tests/test_server.py | 4 +- 4 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/websockets/connection.py b/src/websockets/connection.py index 4a75bede9..aeb774f00 100644 --- a/src/websockets/connection.py +++ b/src/websockets/connection.py @@ -126,7 +126,7 @@ def receive_data(self, data: bytes) -> None: After calling this method: - - You must call :meth:`bytes_to_send` and send this data. + - You must call :meth:`data_to_send` and send this data. - You should call :meth:`events_received` and process these events. """ @@ -139,7 +139,7 @@ def receive_eof(self) -> None: After calling this method: - - You must call :meth:`bytes_to_send` and send this data. + - You must call :meth:`data_to_send` and send this data. - You shouldn't call :meth:`events_received` as it won't return any new events. @@ -228,7 +228,7 @@ def events_received(self) -> List[Event]: # Public API for getting outgoing data after receiving data or sending events. - def bytes_to_send(self) -> List[bytes]: + def data_to_send(self) -> List[bytes]: """ Return data to write to the connection. diff --git a/tests/test_client.py b/tests/test_client.py index 7a78ee09b..747594bf3 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -27,7 +27,7 @@ def test_send_connect(self): self.assertIsInstance(request, Request) client.send_request(request) self.assertEqual( - client.bytes_to_send(), + client.data_to_send(), [ f"GET /test HTTP/1.1\r\n" f"Host: example.com\r\n" diff --git a/tests/test_connection.py b/tests/test_connection.py index d47147d64..3e39a3f9e 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -35,7 +35,7 @@ def assertFrameSent(self, connection, frame, eof=False): mask=connection.side is Side.CLIENT, extensions=connection.extensions, ) - for write in connection.bytes_to_send() + for write in connection.data_to_send() ] frames_expected = [] if frame is None else [frame] if eof: @@ -101,12 +101,12 @@ def test_client_sends_masked_frame(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\xff\x00\xff"): client.send_text(b"Spam", True) - self.assertEqual(client.bytes_to_send(), [self.masked_text_frame_data]) + self.assertEqual(client.data_to_send(), [self.masked_text_frame_data]) def test_server_sends_unmasked_frame(self): server = Connection(Side.SERVER) server.send_text(b"Spam", True) - self.assertEqual(server.bytes_to_send(), [self.unmasked_text_frame_date]) + self.assertEqual(server.data_to_send(), [self.unmasked_text_frame_date]) def test_client_receives_unmasked_frame(self): client = Connection(Side.CLIENT) @@ -178,7 +178,7 @@ def test_client_sends_continuation_after_sending_close(self): # this is the same test as test_client_sends_unexpected_continuation. with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_close(1001) - self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + self.assertEqual(client.data_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) with self.assertRaises(ProtocolError) as raised: client.send_continuation(b"", fin=False) self.assertEqual(str(raised.exception), "unexpected continuation frame") @@ -189,7 +189,7 @@ def test_server_sends_continuation_after_sending_close(self): # this is the same test as test_server_sends_unexpected_continuation. server = Connection(Side.SERVER) server.send_close(1000) - self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x02\x03\xe8", b""]) with self.assertRaises(ProtocolError) as raised: server.send_continuation(b"", fin=False) self.assertEqual(str(raised.exception), "unexpected continuation frame") @@ -222,13 +222,13 @@ def test_client_sends_text(self): with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_text("😀".encode()) self.assertEqual( - client.bytes_to_send(), [b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80"] + client.data_to_send(), [b"\x81\x84\x00\x00\x00\x00\xf0\x9f\x98\x80"] ) def test_server_sends_text(self): server = Connection(Side.SERVER) server.send_text("😀".encode()) - self.assertEqual(server.bytes_to_send(), [b"\x81\x04\xf0\x9f\x98\x80"]) + self.assertEqual(server.data_to_send(), [b"\x81\x04\xf0\x9f\x98\x80"]) def test_client_receives_text(self): client = Connection(Side.CLIENT) @@ -280,24 +280,24 @@ def test_client_sends_fragmented_text(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_text("😀".encode()[:2], fin=False) - self.assertEqual(client.bytes_to_send(), [b"\x01\x82\x00\x00\x00\x00\xf0\x9f"]) + self.assertEqual(client.data_to_send(), [b"\x01\x82\x00\x00\x00\x00\xf0\x9f"]) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_continuation("😀😀".encode()[2:6], fin=False) self.assertEqual( - client.bytes_to_send(), [b"\x00\x84\x00\x00\x00\x00\x98\x80\xf0\x9f"] + client.data_to_send(), [b"\x00\x84\x00\x00\x00\x00\x98\x80\xf0\x9f"] ) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_continuation("😀".encode()[2:], fin=True) - self.assertEqual(client.bytes_to_send(), [b"\x80\x82\x00\x00\x00\x00\x98\x80"]) + self.assertEqual(client.data_to_send(), [b"\x80\x82\x00\x00\x00\x00\x98\x80"]) def test_server_sends_fragmented_text(self): server = Connection(Side.SERVER) server.send_text("😀".encode()[:2], fin=False) - self.assertEqual(server.bytes_to_send(), [b"\x01\x02\xf0\x9f"]) + self.assertEqual(server.data_to_send(), [b"\x01\x02\xf0\x9f"]) server.send_continuation("😀😀".encode()[2:6], fin=False) - self.assertEqual(server.bytes_to_send(), [b"\x00\x04\x98\x80\xf0\x9f"]) + self.assertEqual(server.data_to_send(), [b"\x00\x04\x98\x80\xf0\x9f"]) server.send_continuation("😀".encode()[2:], fin=True) - self.assertEqual(server.bytes_to_send(), [b"\x80\x02\x98\x80"]) + self.assertEqual(server.data_to_send(), [b"\x80\x02\x98\x80"]) def test_client_receives_fragmented_text(self): client = Connection(Side.CLIENT) @@ -437,14 +437,14 @@ def test_client_sends_text_after_sending_close(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_close(1001) - self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + self.assertEqual(client.data_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) with self.assertRaises(InvalidState): client.send_text(b"") def test_server_sends_text_after_sending_close(self): server = Connection(Side.SERVER) server.send_close(1000) - self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x02\x03\xe8", b""]) with self.assertRaises(InvalidState): server.send_text(b"") @@ -476,13 +476,13 @@ def test_client_sends_binary(self): with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_binary(b"\x01\x02\xfe\xff") self.assertEqual( - client.bytes_to_send(), [b"\x82\x84\x00\x00\x00\x00\x01\x02\xfe\xff"] + client.data_to_send(), [b"\x82\x84\x00\x00\x00\x00\x01\x02\xfe\xff"] ) def test_server_sends_binary(self): server = Connection(Side.SERVER) server.send_binary(b"\x01\x02\xfe\xff") - self.assertEqual(server.bytes_to_send(), [b"\x82\x04\x01\x02\xfe\xff"]) + self.assertEqual(server.data_to_send(), [b"\x82\x04\x01\x02\xfe\xff"]) def test_client_receives_binary(self): client = Connection(Side.CLIENT) @@ -518,24 +518,24 @@ def test_client_sends_fragmented_binary(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_binary(b"\x01\x02", fin=False) - self.assertEqual(client.bytes_to_send(), [b"\x02\x82\x00\x00\x00\x00\x01\x02"]) + self.assertEqual(client.data_to_send(), [b"\x02\x82\x00\x00\x00\x00\x01\x02"]) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_continuation(b"\xee\xff\x01\x02", fin=False) self.assertEqual( - client.bytes_to_send(), [b"\x00\x84\x00\x00\x00\x00\xee\xff\x01\x02"] + client.data_to_send(), [b"\x00\x84\x00\x00\x00\x00\xee\xff\x01\x02"] ) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_continuation(b"\xee\xff", fin=True) - self.assertEqual(client.bytes_to_send(), [b"\x80\x82\x00\x00\x00\x00\xee\xff"]) + self.assertEqual(client.data_to_send(), [b"\x80\x82\x00\x00\x00\x00\xee\xff"]) def test_server_sends_fragmented_binary(self): server = Connection(Side.SERVER) server.send_binary(b"\x01\x02", fin=False) - self.assertEqual(server.bytes_to_send(), [b"\x02\x02\x01\x02"]) + self.assertEqual(server.data_to_send(), [b"\x02\x02\x01\x02"]) server.send_continuation(b"\xee\xff\x01\x02", fin=False) - self.assertEqual(server.bytes_to_send(), [b"\x00\x04\xee\xff\x01\x02"]) + self.assertEqual(server.data_to_send(), [b"\x00\x04\xee\xff\x01\x02"]) server.send_continuation(b"\xee\xff", fin=True) - self.assertEqual(server.bytes_to_send(), [b"\x80\x02\xee\xff"]) + self.assertEqual(server.data_to_send(), [b"\x80\x02\xee\xff"]) def test_client_receives_fragmented_binary(self): client = Connection(Side.CLIENT) @@ -639,14 +639,14 @@ def test_client_sends_binary_after_sending_close(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_close(1001) - self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + self.assertEqual(client.data_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) with self.assertRaises(InvalidState): client.send_binary(b"") def test_server_sends_binary_after_sending_close(self): server = Connection(Side.SERVER) server.send_close(1000) - self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x02\x03\xe8", b""]) with self.assertRaises(InvalidState): server.send_binary(b"") @@ -677,13 +677,13 @@ def test_client_sends_close(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x3c\x3c\x3c\x3c"): client.send_close() - self.assertEqual(client.bytes_to_send(), [b"\x88\x80\x3c\x3c\x3c\x3c"]) + self.assertEqual(client.data_to_send(), [b"\x88\x80\x3c\x3c\x3c\x3c"]) self.assertIs(client.state, State.CLOSING) def test_server_sends_close(self): server = Connection(Side.SERVER) server.send_close() - self.assertEqual(server.bytes_to_send(), [b"\x88\x00", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x00", b""]) self.assertIs(server.state, State.CLOSING) def test_client_receives_close(self): @@ -691,14 +691,14 @@ def test_client_receives_close(self): with self.enforce_mask(b"\x3c\x3c\x3c\x3c"): client.receive_data(b"\x88\x00") self.assertEqual(client.events_received(), [Frame(True, OP_CLOSE, b"")]) - self.assertEqual(client.bytes_to_send(), [b"\x88\x80\x3c\x3c\x3c\x3c"]) + self.assertEqual(client.data_to_send(), [b"\x88\x80\x3c\x3c\x3c\x3c"]) self.assertIs(client.state, State.CLOSING) def test_server_receives_close(self): server = Connection(Side.SERVER) server.receive_data(b"\x88\x80\x3c\x3c\x3c\x3c") self.assertEqual(server.events_received(), [Frame(True, OP_CLOSE, b"")]) - self.assertEqual(server.bytes_to_send(), [b"\x88\x00", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x00", b""]) self.assertIs(server.state, State.CLOSING) def test_client_sends_close_then_receives_close(self): @@ -761,13 +761,13 @@ def test_client_sends_close_with_code(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_close(1001) - self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + self.assertEqual(client.data_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) self.assertIs(client.state, State.CLOSING) def test_server_sends_close_with_code(self): server = Connection(Side.SERVER) server.send_close(1000) - self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x02\x03\xe8", b""]) self.assertIs(server.state, State.CLOSING) def test_client_receives_close_with_code(self): @@ -787,14 +787,14 @@ def test_client_sends_close_with_code_and_reason(self): with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_close(1001, "going away") self.assertEqual( - client.bytes_to_send(), [b"\x88\x8c\x00\x00\x00\x00\x03\xe9going away"] + client.data_to_send(), [b"\x88\x8c\x00\x00\x00\x00\x03\xe9going away"] ) self.assertIs(client.state, State.CLOSING) def test_server_sends_close_with_code_and_reason(self): server = Connection(Side.SERVER) server.send_close(1000, "OK") - self.assertEqual(server.bytes_to_send(), [b"\x88\x04\x03\xe8OK", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x04\x03\xe8OK", b""]) self.assertIs(server.state, State.CLOSING) def test_client_receives_close_with_code_and_reason(self): @@ -870,12 +870,12 @@ def test_client_sends_ping(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x44\x88\xcc"): client.send_ping(b"") - self.assertEqual(client.bytes_to_send(), [b"\x89\x80\x00\x44\x88\xcc"]) + self.assertEqual(client.data_to_send(), [b"\x89\x80\x00\x44\x88\xcc"]) def test_server_sends_ping(self): server = Connection(Side.SERVER) server.send_ping(b"") - self.assertEqual(server.bytes_to_send(), [b"\x89\x00"]) + self.assertEqual(server.data_to_send(), [b"\x89\x00"]) def test_client_receives_ping(self): client = Connection(Side.CLIENT) @@ -906,13 +906,13 @@ def test_client_sends_ping_with_data(self): with self.enforce_mask(b"\x00\x44\x88\xcc"): client.send_ping(b"\x22\x66\xaa\xee") self.assertEqual( - client.bytes_to_send(), [b"\x89\x84\x00\x44\x88\xcc\x22\x22\x22\x22"] + client.data_to_send(), [b"\x89\x84\x00\x44\x88\xcc\x22\x22\x22\x22"] ) def test_server_sends_ping_with_data(self): server = Connection(Side.SERVER) server.send_ping(b"\x22\x66\xaa\xee") - self.assertEqual(server.bytes_to_send(), [b"\x89\x04\x22\x66\xaa\xee"]) + self.assertEqual(server.data_to_send(), [b"\x89\x04\x22\x66\xaa\xee"]) def test_client_receives_ping_with_data(self): client = Connection(Side.CLIENT) @@ -970,7 +970,7 @@ def test_client_sends_ping_after_sending_close(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_close(1001) - self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + self.assertEqual(client.data_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) # The spec says: "An endpoint MAY send a Ping frame any time (...) # before the connection is closed" but websockets doesn't support # sending a Ping frame after a Close frame. @@ -983,7 +983,7 @@ def test_client_sends_ping_after_sending_close(self): def test_server_sends_ping_after_sending_close(self): server = Connection(Side.SERVER) server.send_close(1000) - self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x02\x03\xe8", b""]) # The spec says: "An endpoint MAY send a Ping frame any time (...) # before the connection is closed" but websockets doesn't support # sending a Ping frame after a Close frame. @@ -1026,12 +1026,12 @@ def test_client_sends_pong(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x44\x88\xcc"): client.send_pong(b"") - self.assertEqual(client.bytes_to_send(), [b"\x8a\x80\x00\x44\x88\xcc"]) + self.assertEqual(client.data_to_send(), [b"\x8a\x80\x00\x44\x88\xcc"]) def test_server_sends_pong(self): server = Connection(Side.SERVER) server.send_pong(b"") - self.assertEqual(server.bytes_to_send(), [b"\x8a\x00"]) + self.assertEqual(server.data_to_send(), [b"\x8a\x00"]) def test_client_receives_pong(self): client = Connection(Side.CLIENT) @@ -1054,13 +1054,13 @@ def test_client_sends_pong_with_data(self): with self.enforce_mask(b"\x00\x44\x88\xcc"): client.send_pong(b"\x22\x66\xaa\xee") self.assertEqual( - client.bytes_to_send(), [b"\x8a\x84\x00\x44\x88\xcc\x22\x22\x22\x22"] + client.data_to_send(), [b"\x8a\x84\x00\x44\x88\xcc\x22\x22\x22\x22"] ) def test_server_sends_pong_with_data(self): server = Connection(Side.SERVER) server.send_pong(b"\x22\x66\xaa\xee") - self.assertEqual(server.bytes_to_send(), [b"\x8a\x04\x22\x66\xaa\xee"]) + self.assertEqual(server.data_to_send(), [b"\x8a\x04\x22\x66\xaa\xee"]) def test_client_receives_pong_with_data(self): client = Connection(Side.CLIENT) @@ -1110,7 +1110,7 @@ def test_client_sends_pong_after_sending_close(self): client = Connection(Side.CLIENT) with self.enforce_mask(b"\x00\x00\x00\x00"): client.send_close(1001) - self.assertEqual(client.bytes_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) + self.assertEqual(client.data_to_send(), [b"\x88\x82\x00\x00\x00\x00\x03\xe9"]) # websockets doesn't support sending a Pong frame after a Close frame. with self.assertRaises(InvalidState): client.send_pong(b"") @@ -1118,7 +1118,7 @@ def test_client_sends_pong_after_sending_close(self): def test_server_sends_pong_after_sending_close(self): server = Connection(Side.SERVER) server.send_close(1000) - self.assertEqual(server.bytes_to_send(), [b"\x88\x02\x03\xe8", b""]) + self.assertEqual(server.data_to_send(), [b"\x88\x02\x03\xe8", b""]) # websockets doesn't support sending a Pong frame after a Close frame. with self.assertRaises(InvalidState): server.send_pong(b"") @@ -1465,13 +1465,13 @@ def test_client_extension_encodes_frame(self): client.extensions = [Rsv2Extension()] with self.enforce_mask(b"\x00\x44\x88\xcc"): client.send_ping(b"") - self.assertEqual(client.bytes_to_send(), [b"\xa9\x80\x00\x44\x88\xcc"]) + self.assertEqual(client.data_to_send(), [b"\xa9\x80\x00\x44\x88\xcc"]) def test_server_extension_encodes_frame(self): server = Connection(Side.SERVER) server.extensions = [Rsv2Extension()] server.send_ping(b"") - self.assertEqual(server.bytes_to_send(), [b"\xa9\x00"]) + self.assertEqual(server.data_to_send(), [b"\xa9\x00"]) def test_client_extension_decodes_frame(self): client = Connection(Side.CLIENT) diff --git a/tests/test_server.py b/tests/test_server.py index a180b08e2..ad56a37bc 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -91,7 +91,7 @@ def test_send_accept(self): self.assertIsInstance(response, Response) server.send_response(response) self.assertEqual( - server.bytes_to_send(), + server.data_to_send(), [ f"HTTP/1.1 101 Switching Protocols\r\n" f"Upgrade: websocket\r\n" @@ -111,7 +111,7 @@ def test_send_reject(self): self.assertIsInstance(response, Response) server.send_response(response) self.assertEqual( - server.bytes_to_send(), + server.data_to_send(), [ f"HTTP/1.1 404 Not Found\r\n" f"Date: {DATE}\r\n" From 44a5453612e7020d1305355c74c3d08ee4db4e91 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 28 Nov 2020 15:16:46 +0100 Subject: [PATCH 058/104] Extract logic for auto-configuring compression. --- src/websockets/asyncio_client.py | 12 +-- src/websockets/asyncio_server.py | 10 +- .../extensions/permessage_deflate.py | 45 +++++++++ tests/extensions/test_base.py | 36 +++++++ tests/extensions/test_permessage_deflate.py | 94 +++++++++++++++++++ tests/test_asyncio_client_server.py | 66 +------------ 6 files changed, 184 insertions(+), 79 deletions(-) diff --git a/src/websockets/asyncio_client.py b/src/websockets/asyncio_client.py index 43e3c1cd2..d22ba764a 100644 --- a/src/websockets/asyncio_client.py +++ b/src/websockets/asyncio_client.py @@ -22,7 +22,7 @@ SecurityError, ) from .extensions.base import ClientExtensionFactory, Extension -from .extensions.permessage_deflate import ClientPerMessageDeflateFactory +from .extensions.permessage_deflate import enable_client_permessage_deflate from .handshake_legacy import build_request, check_response from .headers import ( build_authorization_basic, @@ -425,15 +425,7 @@ def __init__( ) if compression == "deflate": - if extensions is None: - extensions = [] - if not any( - extension_factory.name == ClientPerMessageDeflateFactory.name - for extension_factory in extensions - ): - extensions = list(extensions) + [ - ClientPerMessageDeflateFactory(client_max_window_bits=True) - ] + extensions = enable_client_permessage_deflate(extensions) elif compression is not None: raise ValueError(f"unsupported compression: {compression}") diff --git a/src/websockets/asyncio_server.py b/src/websockets/asyncio_server.py index b4f7fbc92..79ceddf4b 100644 --- a/src/websockets/asyncio_server.py +++ b/src/websockets/asyncio_server.py @@ -39,7 +39,7 @@ NegotiationError, ) from .extensions.base import Extension, ServerExtensionFactory -from .extensions.permessage_deflate import ServerPerMessageDeflateFactory +from .extensions.permessage_deflate import enable_server_permessage_deflate from .handshake_legacy import build_response, check_request from .headers import build_extension, parse_extension, parse_subprotocol from .http import USER_AGENT @@ -903,13 +903,7 @@ def __init__( secure = kwargs.get("ssl") is not None if compression == "deflate": - if extensions is None: - extensions = [] - if not any( - ext_factory.name == ServerPerMessageDeflateFactory.name - for ext_factory in extensions - ): - extensions = list(extensions) + [ServerPerMessageDeflateFactory()] + extensions = enable_server_permessage_deflate(extensions) elif compression is not None: raise ValueError(f"unsupported compression: {compression}") diff --git a/src/websockets/extensions/permessage_deflate.py b/src/websockets/extensions/permessage_deflate.py index 184183061..9a3fc4ba5 100644 --- a/src/websockets/extensions/permessage_deflate.py +++ b/src/websockets/extensions/permessage_deflate.py @@ -22,7 +22,9 @@ __all__ = [ "PerMessageDeflate", "ClientPerMessageDeflateFactory", + "enable_client_permessage_deflate", "ServerPerMessageDeflateFactory", + "enable_server_permessage_deflate", ] _EMPTY_UNCOMPRESSED_BLOCK = b"\x00\x00\xff\xff" @@ -424,6 +426,29 @@ def process_response_params( ) +def enable_client_permessage_deflate( + extensions: Optional[Sequence[ClientExtensionFactory]], +) -> Sequence[ClientExtensionFactory]: + """ + Enable Per-Message Deflate with default settings in client extensions. + + If the extension is already present, perhaps with non-default settings, + the configuration isn't changed. + + + """ + if extensions is None: + extensions = [] + if not any( + extension_factory.name == ClientPerMessageDeflateFactory.name + for extension_factory in extensions + ): + extensions = list(extensions) + [ + ClientPerMessageDeflateFactory(client_max_window_bits=True) + ] + return extensions + + class ServerPerMessageDeflateFactory(ServerExtensionFactory): """ Server-side extension factory for the Per-Message Deflate extension. @@ -584,3 +609,23 @@ def process_request_params( self.compress_settings, ), ) + + +def enable_server_permessage_deflate( + extensions: Optional[Sequence[ServerExtensionFactory]], +) -> Sequence[ServerExtensionFactory]: + """ + Enable Per-Message Deflate with default settings in server extensions. + + If the extension is already present, perhaps with non-default settings, + the configuration isn't changed. + + """ + if extensions is None: + extensions = [] + if not any( + ext_factory.name == ServerPerMessageDeflateFactory.name + for ext_factory in extensions + ): + extensions = list(extensions) + [ServerPerMessageDeflateFactory()] + return extensions diff --git a/tests/extensions/test_base.py b/tests/extensions/test_base.py index ba8657b65..0daa34211 100644 --- a/tests/extensions/test_base.py +++ b/tests/extensions/test_base.py @@ -1,4 +1,40 @@ +from websockets.exceptions import NegotiationError from websockets.extensions.base import * # noqa # Abstract classes don't provide any behavior to test. + + +class ClientNoOpExtensionFactory: + name = "x-no-op" + + def get_request_params(self): + return [] + + def process_response_params(self, params, accepted_extensions): + if params: + raise NegotiationError() + return NoOpExtension() + + +class ServerNoOpExtensionFactory: + name = "x-no-op" + + def __init__(self, params=None): + self.params = params or [] + + def process_request_params(self, params, accepted_extensions): + return self.params, NoOpExtension() + + +class NoOpExtension: + name = "x-no-op" + + def __repr__(self): + return "NoOpExtension()" + + def decode(self, frame, *, max_size=None): + return frame + + def encode(self, frame): + return frame diff --git a/tests/extensions/test_permessage_deflate.py b/tests/extensions/test_permessage_deflate.py index f9fca1999..328861e58 100644 --- a/tests/extensions/test_permessage_deflate.py +++ b/tests/extensions/test_permessage_deflate.py @@ -20,6 +20,8 @@ serialize_close, ) +from .test_base import ClientNoOpExtensionFactory, ServerNoOpExtensionFactory + class ExtensionTestsMixin: def assertExtensionEqual(self, extension1, extension2): @@ -500,6 +502,52 @@ def test_process_response_params_deduplication(self): [], [PerMessageDeflate(False, False, 15, 15)] ) + def test_enable_client_permessage_deflate(self): + for extensions, ( + expected_len, + expected_position, + expected_compress_settings, + ) in [ + ( + None, + (1, 0, None), + ), + ( + [], + (1, 0, None), + ), + ( + [ClientNoOpExtensionFactory()], + (2, 1, None), + ), + ( + [ClientPerMessageDeflateFactory(compress_settings={"level": 1})], + (1, 0, {"level": 1}), + ), + ( + [ + ClientPerMessageDeflateFactory(compress_settings={"level": 1}), + ClientNoOpExtensionFactory(), + ], + (2, 0, {"level": 1}), + ), + ( + [ + ClientNoOpExtensionFactory(), + ClientPerMessageDeflateFactory(compress_settings={"level": 1}), + ], + (2, 1, {"level": 1}), + ), + ]: + with self.subTest(extensions=extensions): + extensions = enable_client_permessage_deflate(extensions) + self.assertEqual(len(extensions), expected_len) + extension = extensions[expected_position] + self.assertIsInstance(extension, ClientPerMessageDeflateFactory) + self.assertEqual( + extension.compress_settings, expected_compress_settings + ) + class ServerPerMessageDeflateFactoryTests(unittest.TestCase, ExtensionTestsMixin): def test_name(self): @@ -790,3 +838,49 @@ def test_process_response_params_deduplication(self): factory.process_request_params( [], [PerMessageDeflate(False, False, 15, 15)] ) + + def test_enable_server_permessage_deflate(self): + for extensions, ( + expected_len, + expected_position, + expected_compress_settings, + ) in [ + ( + None, + (1, 0, None), + ), + ( + [], + (1, 0, None), + ), + ( + [ServerNoOpExtensionFactory()], + (2, 1, None), + ), + ( + [ServerPerMessageDeflateFactory(compress_settings={"level": 1})], + (1, 0, {"level": 1}), + ), + ( + [ + ServerPerMessageDeflateFactory(compress_settings={"level": 1}), + ServerNoOpExtensionFactory(), + ], + (2, 0, {"level": 1}), + ), + ( + [ + ServerNoOpExtensionFactory(), + ServerPerMessageDeflateFactory(compress_settings={"level": 1}), + ], + (2, 1, {"level": 1}), + ), + ]: + with self.subTest(extensions=extensions): + extensions = enable_server_permessage_deflate(extensions) + self.assertEqual(len(extensions), expected_len) + extension = extensions[expected_position] + self.assertIsInstance(extension, ServerPerMessageDeflateFactory) + self.assertEqual( + extension.compress_settings, expected_compress_settings + ) diff --git a/tests/test_asyncio_client_server.py b/tests/test_asyncio_client_server.py index cff76d1f2..76c29334e 100644 --- a/tests/test_asyncio_client_server.py +++ b/tests/test_asyncio_client_server.py @@ -34,6 +34,11 @@ from websockets.protocol import State from websockets.uri import parse_uri +from .extensions.test_base import ( + ClientNoOpExtensionFactory, + NoOpExtension, + ServerNoOpExtensionFactory, +) from .test_protocol import MS from .utils import AsyncioTestCase @@ -188,41 +193,6 @@ class BarClientProtocol(WebSocketClientProtocol): pass -class ClientNoOpExtensionFactory: - name = "x-no-op" - - def get_request_params(self): - return [] - - def process_response_params(self, params, accepted_extensions): - if params: - raise NegotiationError() - return NoOpExtension() - - -class ServerNoOpExtensionFactory: - name = "x-no-op" - - def __init__(self, params=None): - self.params = params or [] - - def process_request_params(self, params, accepted_extensions): - return self.params, NoOpExtension() - - -class NoOpExtension: - name = "x-no-op" - - def __repr__(self): - return "NoOpExtension()" - - def decode(self, frame, *, max_size=None): - return frame - - def encode(self, frame): - return frame - - class ClientServerTestsMixin: secure = False @@ -974,32 +944,6 @@ def test_compression_deflate(self): repr([PerMessageDeflate(False, False, 15, 15)]), ) - @with_server( - extensions=[ - ServerPerMessageDeflateFactory( - client_no_context_takeover=True, server_max_window_bits=10 - ) - ], - compression="deflate", # overridden by explicit config - ) - @with_client( - "/extensions", - extensions=[ - ClientPerMessageDeflateFactory( - server_no_context_takeover=True, client_max_window_bits=12 - ) - ], - compression="deflate", # overridden by explicit config - ) - def test_compression_deflate_and_explicit_config(self): - server_extensions = self.loop.run_until_complete(self.client.recv()) - self.assertEqual( - server_extensions, repr([PerMessageDeflate(True, True, 12, 10)]) - ) - self.assertEqual( - repr(self.client.extensions), repr([PerMessageDeflate(True, True, 10, 12)]) - ) - def test_compression_unsupported_server(self): with self.assertRaises(ValueError): self.start_server(compression="xz") From 3f36975b197f1250258055d403d2061f70013278 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 28 Nov 2020 15:18:44 +0100 Subject: [PATCH 059/104] Name asyncio protocol consistently. This isn't comparable to ws_server on the server side. --- src/websockets/asyncio_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/websockets/asyncio_client.py b/src/websockets/asyncio_client.py index d22ba764a..3f406170a 100644 --- a/src/websockets/asyncio_client.py +++ b/src/websockets/asyncio_client.py @@ -517,7 +517,7 @@ async def __aexit__( exc_value: Optional[BaseException], traceback: Optional[TracebackType], ) -> None: - await self.ws_client.close() + await self.protocol.close() # await connect(...) @@ -546,7 +546,7 @@ async def __await_impl__(self) -> WebSocketClientProtocol: await protocol.wait_closed() raise else: - self.ws_client = protocol + self.protocol = protocol return protocol except RedirectHandshake as exc: self.handle_redirect(exc.uri) From 32b95fb0dd2cfc07d38df45dcf7f0ebf05008424 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 28 Nov 2020 15:19:11 +0100 Subject: [PATCH 060/104] Name pong waiter consistently. --- src/websockets/protocol.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index ba4fc1d3c..1552fb060 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -291,7 +291,7 @@ def __init__( # Protect sending fragmented messages. self._fragmented_message_waiter: Optional[asyncio.Future[None]] = None - # Mapping of ping IDs to waiters, in chronological order. + # Mapping of ping IDs to pong waiters, in chronological order. self.pings: Dict[bytes, asyncio.Future[None]] = {} # Task running the data transfer. @@ -736,15 +736,15 @@ async def ping(self, data: Optional[Data] = None) -> Awaitable[None]: """ Send a ping. - Return a :class:`~asyncio.Future` which will be completed when the - corresponding pong is received and which you may ignore if you don't - want to wait. + Return a :class:`~asyncio.Future` that will be completed when the + corresponding pong is received. You can ignore it if you don't intend + to wait. A ping may serve as a keepalive or as a check that the remote endpoint received all messages up to this point:: pong_waiter = await ws.ping() - await pong_waiter # only if you want to wait for the pong + await pong_waiter # only if you want to wait for the pong By default, the ping contains four random bytes. This payload may be overridden with the optional ``data`` argument which must be a string @@ -1155,12 +1155,12 @@ async def keepalive_ping(self) -> None: # ping() raises ConnectionClosed if the connection is lost, # when connection_lost() calls abort_pings(). - ping_waiter = await self.ping() + pong_waiter = await self.ping() if self.ping_timeout is not None: try: await asyncio.wait_for( - ping_waiter, + pong_waiter, self.ping_timeout, loop=self.loop if sys.version_info[:2] < (3, 8) else None, ) From 165d0c69548e4c9d02624bcbb6eb565bb4c0c136 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 28 Nov 2020 19:15:43 +0100 Subject: [PATCH 061/104] Move asyncio-based APIs to a legacy subpackage. Clean up deprecations in the process. --- docs/api.rst | 8 +- docs/changelog.rst | 136 +- docs/cheatsheet.rst | 36 +- docs/deployment.rst | 6 +- docs/design.rst | 178 +- docs/extensions.rst | 5 +- docs/faq.rst | 8 +- docs/intro.rst | 2 +- setup.py | 2 +- src/websockets/__init__.py | 10 +- src/websockets/__main__.py | 2 +- src/websockets/auth.py | 163 +- src/websockets/client.py | 11 +- src/websockets/exceptions.py | 2 +- src/websockets/framing.py | 135 +- src/websockets/handshake.py | 8 +- src/websockets/http.py | 4 +- src/websockets/legacy/__init__.py | 0 src/websockets/legacy/auth.py | 165 ++ .../{asyncio_client.py => legacy/client.py} | 22 +- src/websockets/legacy/framing.py | 135 ++ .../handshake.py} | 12 +- .../{http_legacy.py => legacy/http.py} | 4 +- src/websockets/legacy/protocol.py | 1459 ++++++++++++++++ .../{asyncio_server.py => legacy/server.py} | 32 +- src/websockets/protocol.py | 1466 +--------------- src/websockets/server.py | 15 +- tests/__init__.py | 10 - tests/legacy/__init__.py | 0 tests/legacy/test_auth.py | 160 ++ .../test_client_server.py} | 39 +- tests/legacy/test_framing.py | 171 ++ .../test_handshake.py} | 2 +- .../test_http.py} | 4 +- tests/legacy/test_protocol.py | 1489 ++++++++++++++++ tests/legacy/utils.py | 93 + tests/test_auth.py | 162 +- tests/test_exports.py | 6 +- tests/test_framing.py | 174 +- tests/test_protocol.py | 1491 +---------------- tests/utils.py | 92 - 41 files changed, 3964 insertions(+), 3955 deletions(-) create mode 100644 src/websockets/legacy/__init__.py create mode 100644 src/websockets/legacy/auth.py rename src/websockets/{asyncio_client.py => legacy/client.py} (97%) create mode 100644 src/websockets/legacy/framing.py rename src/websockets/{handshake_legacy.py => legacy/handshake.py} (93%) rename src/websockets/{http_legacy.py => legacy/http.py} (98%) create mode 100644 src/websockets/legacy/protocol.py rename src/websockets/{asyncio_server.py => legacy/server.py} (97%) create mode 100644 tests/legacy/__init__.py create mode 100644 tests/legacy/test_auth.py rename tests/{test_asyncio_client_server.py => legacy/test_client_server.py} (97%) create mode 100644 tests/legacy/test_framing.py rename tests/{test_handshake_legacy.py => legacy/test_handshake.py} (99%) rename tests/{test_http_legacy.py => legacy/test_http.py} (98%) create mode 100644 tests/legacy/test_protocol.py create mode 100644 tests/legacy/utils.py diff --git a/docs/api.rst b/docs/api.rst index b4bddaf38..c73cf59d3 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -29,7 +29,7 @@ High-level Server ...... -.. automodule:: websockets.server +.. automodule:: websockets.legacy.server .. autofunction:: serve(ws_handler, host=None, port=None, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origins=None, extensions=None, subprotocols=None, extra_headers=None, process_request=None, select_subprotocol=None, **kwds) :async: @@ -53,7 +53,7 @@ Server Client ...... -.. automodule:: websockets.client +.. automodule:: websockets.legacy.client .. autofunction:: connect(uri, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origin=None, extensions=None, subprotocols=None, extra_headers=None, **kwds) :async: @@ -68,7 +68,7 @@ Client Shared ...... -.. automodule:: websockets.protocol +.. automodule:: websockets.legacy.protocol .. autoclass:: WebSocketCommonProtocol(*, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None) @@ -107,7 +107,7 @@ Per-Message Deflate Extension HTTP Basic Auth ............... -.. automodule:: websockets.auth +.. automodule:: websockets.legacy.auth .. autofunction:: basic_auth_protocol_factory diff --git a/docs/changelog.rst b/docs/changelog.rst index c131f0528..291ec6938 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,17 +10,23 @@ Changelog .. note:: - **Version 9.0 moves or deprecates several low-level APIs.** + **Version 9.0 moves or deprecates several APIs.** * Import :class:`~datastructures.Headers` and :exc:`~datastructures.MultipleValuesError` from :mod:`websockets.datastructures` instead of :mod:`websockets.http`. + * :mod:`websockets.client`, :mod:`websockets.server,` + :mod:`websockets.protocol`, and :mod:`websockets.auth` were moved to + :mod:`websockets.legacy.client`, :mod:`websockets.legacy.server`, + :mod:`websockets.legacy.protocol`, and :mod:`websockets.legacy.auth` + respectively. + * :mod:`websockets.handshake` is deprecated. * :mod:`websockets.http` is deprecated. - * :mod:`websocket.framing` is deprecated. + * :mod:`websockets.framing` is deprecated. Aliases provide backwards compatibility for all previously public APIs. @@ -37,7 +43,7 @@ Changelog ..... * Restored the ability to pass a socket with the ``sock`` parameter of - :func:`~server.serve`. + :func:`~legacy.server.serve`. * Removed an incorrect assertion when a connection drops. @@ -60,9 +66,9 @@ Changelog Previously, it could be a function or a coroutine. - If you're passing a ``process_request`` argument to :func:`~server.serve` - or :class:`~server.WebSocketServerProtocol`, or if you're overriding - :meth:`~protocol.WebSocketServerProtocol.process_request` in a subclass, + If you're passing a ``process_request`` argument to :func:`~legacy.server.serve` + or :class:`~legacy.server.WebSocketServerProtocol`, or if you're overriding + :meth:`~legacy.server.WebSocketServerProtocol.process_request` in a subclass, define it with ``async def`` instead of ``def``. For backwards compatibility, functions are still mostly supported, but @@ -78,10 +84,10 @@ Changelog .. note:: **Version 8.0 deprecates the** ``host`` **,** ``port`` **, and** ``secure`` - **attributes of** :class:`~protocol.WebSocketCommonProtocol`. + **attributes of** :class:`~legacy.protocol.WebSocketCommonProtocol`. - Use :attr:`~protocol.WebSocketCommonProtocol.local_address` in servers and - :attr:`~protocol.WebSocketCommonProtocol.remote_address` in clients + Use :attr:`~legacy.protocol.WebSocketCommonProtocol.local_address` in servers and + :attr:`~legacy.protocol.WebSocketCommonProtocol.remote_address` in clients instead of ``host`` and ``port``. .. note:: @@ -98,9 +104,9 @@ Changelog Also: -* :meth:`~protocol.WebSocketCommonProtocol.send`, - :meth:`~protocol.WebSocketCommonProtocol.ping`, and - :meth:`~protocol.WebSocketCommonProtocol.pong` support bytes-like types +* :meth:`~legacy.protocol.WebSocketCommonProtocol.send`, + :meth:`~legacy.protocol.WebSocketCommonProtocol.ping`, and + :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` support bytes-like types :class:`bytearray` and :class:`memoryview` in addition to :class:`bytes`. * Added :exc:`~exceptions.ConnectionClosedOK` and @@ -108,18 +114,18 @@ Also: :exc:`~exceptions.ConnectionClosed` to tell apart normal connection termination from errors. -* Added :func:`~auth.basic_auth_protocol_factory` to enforce HTTP Basic Auth +* Added :func:`~legacy.auth.basic_auth_protocol_factory` to enforce HTTP Basic Auth on the server side. -* :func:`~client.connect` handles redirects from the server during the +* :func:`~legacy.client.connect` handles redirects from the server during the handshake. -* :func:`~client.connect` supports overriding ``host`` and ``port``. +* :func:`~legacy.client.connect` supports overriding ``host`` and ``port``. -* Added :func:`~client.unix_connect` for connecting to Unix sockets. +* Added :func:`~legacy.client.unix_connect` for connecting to Unix sockets. * Improved support for sending fragmented messages by accepting asynchronous - iterators in :meth:`~protocol.WebSocketCommonProtocol.send`. + iterators in :meth:`~legacy.protocol.WebSocketCommonProtocol.send`. * Prevented spurious log messages about :exc:`~exceptions.ConnectionClosed` exceptions in keepalive ping task. If you were using ``ping_timeout=None`` @@ -150,7 +156,7 @@ Also: .. warning:: **Version 7.0 renames the** ``timeout`` **argument of** - :func:`~server.serve()` **and** :func:`~client.connect` **to** + :func:`~legacy.server.serve()` **and** :func:`~legacy.client.connect` **to** ``close_timeout`` **.** This prevents confusion with ``ping_timeout``. @@ -160,7 +166,7 @@ Also: .. warning:: **Version 7.0 changes how a server terminates connections when it's - closed with** :meth:`~server.WebSocketServer.close` **.** + closed with** :meth:`~legacy.server.WebSocketServer.close` **.** Previously, connections handlers were canceled. Now, connections are closed with close code 1001 (going away). From the perspective of the @@ -177,7 +183,7 @@ Also: .. note:: - **Version 7.0 changes how a** :meth:`~protocol.WebSocketCommonProtocol.ping` + **Version 7.0 changes how a** :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` **that hasn't received a pong yet behaves when the connection is closed.** The ping — as in ``ping = await websocket.ping()`` — used to be canceled @@ -188,7 +194,7 @@ Also: .. note:: **Version 7.0 raises a** :exc:`RuntimeError` **exception if two coroutines - call** :meth:`~protocol.WebSocketCommonProtocol.recv` **concurrently.** + call** :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` **concurrently.** Concurrent calls lead to non-deterministic behavior because there are no guarantees about which coroutine will receive which message. @@ -197,17 +203,17 @@ Also: * ``websockets`` sends Ping frames at regular intervals and closes the connection if it doesn't receive a matching Pong frame. See - :class:`~protocol.WebSocketCommonProtocol` for details. + :class:`~legacy.protocol.WebSocketCommonProtocol` for details. * Added ``process_request`` and ``select_subprotocol`` arguments to - :func:`~server.serve` and :class:`~server.WebSocketServerProtocol` to - customize :meth:`~server.WebSocketServerProtocol.process_request` and - :meth:`~server.WebSocketServerProtocol.select_subprotocol` without - subclassing :class:`~server.WebSocketServerProtocol`. + :func:`~legacy.server.serve` and :class:`~legacy.server.WebSocketServerProtocol` to + customize :meth:`~legacy.server.WebSocketServerProtocol.process_request` and + :meth:`~legacy.server.WebSocketServerProtocol.select_subprotocol` without + subclassing :class:`~legacy.server.WebSocketServerProtocol`. * Added support for sending fragmented messages. -* Added the :meth:`~protocol.WebSocketCommonProtocol.wait_closed` method to +* Added the :meth:`~legacy.protocol.WebSocketCommonProtocol.wait_closed` method to protocols. * Added an interactive client: ``python -m websockets ``. @@ -215,7 +221,7 @@ Also: * Changed the ``origins`` argument to represent the lack of an origin with ``None`` rather than ``''``. -* Fixed a data loss bug in :meth:`~protocol.WebSocketCommonProtocol.recv`: +* Fixed a data loss bug in :meth:`~legacy.protocol.WebSocketCommonProtocol.recv`: canceling it at the wrong time could result in messages being dropped. * Improved handling of multiple HTTP headers with the same name. @@ -230,18 +236,18 @@ Also: **Version 6.0 introduces the** :class:`~http.Headers` **class for managing HTTP headers and changes several public APIs:** - * :meth:`~server.WebSocketServerProtocol.process_request` now receives a + * :meth:`~legacy.server.WebSocketServerProtocol.process_request` now receives a :class:`~http.Headers` instead of a :class:`~http.client.HTTPMessage` in the ``request_headers`` argument. - * The :attr:`~protocol.WebSocketCommonProtocol.request_headers` and - :attr:`~protocol.WebSocketCommonProtocol.response_headers` attributes of - :class:`~protocol.WebSocketCommonProtocol` are :class:`~http.Headers` + * The :attr:`~legacy.protocol.WebSocketCommonProtocol.request_headers` and + :attr:`~legacy.protocol.WebSocketCommonProtocol.response_headers` attributes of + :class:`~legacy.protocol.WebSocketCommonProtocol` are :class:`~http.Headers` instead of :class:`~http.client.HTTPMessage`. - * The :attr:`~protocol.WebSocketCommonProtocol.raw_request_headers` and - :attr:`~protocol.WebSocketCommonProtocol.raw_response_headers` - attributes of :class:`~protocol.WebSocketCommonProtocol` are removed. + * The :attr:`~legacy.protocol.WebSocketCommonProtocol.raw_request_headers` and + :attr:`~legacy.protocol.WebSocketCommonProtocol.raw_response_headers` + attributes of :class:`~legacy.protocol.WebSocketCommonProtocol` are removed. Use :meth:`~http.Headers.raw_items` instead. * Functions defined in the :mod:`~handshake` module now receive @@ -265,7 +271,7 @@ Also: ..... * Fixed a regression in the 5.0 release that broke some invocations of - :func:`~server.serve()` and :func:`~client.connect`. + :func:`~legacy.server.serve()` and :func:`~legacy.client.connect`. 5.0 ... @@ -290,7 +296,7 @@ Also: Also: -* :func:`~client.connect` performs HTTP Basic Auth when the URI contains +* :func:`~legacy.client.connect` performs HTTP Basic Auth when the URI contains credentials. * Iterating on incoming messages no longer raises an exception when the @@ -299,13 +305,13 @@ Also: * A plain HTTP request now receives a 426 Upgrade Required response and doesn't log a stack trace. -* :func:`~server.unix_serve` can be used as an asynchronous context manager on +* :func:`~legacy.server.unix_serve` can be used as an asynchronous context manager on Python ≥ 3.5.1. -* Added the :attr:`~protocol.WebSocketCommonProtocol.closed` property to +* Added the :attr:`~legacy.protocol.WebSocketCommonProtocol.closed` property to protocols. -* If a :meth:`~protocol.WebSocketCommonProtocol.ping` doesn't receive a pong, +* If a :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` doesn't receive a pong, it's canceled when the connection is closed. * Reported the cause of :exc:`~exceptions.ConnectionClosed` exceptions. @@ -346,7 +352,7 @@ Also: Compression should improve performance but it increases RAM and CPU use. If you want to disable compression, add ``compression=None`` when calling - :func:`~server.serve()` or :func:`~client.connect`. + :func:`~legacy.server.serve()` or :func:`~legacy.client.connect`. .. warning:: @@ -360,13 +366,13 @@ Also: Also: -* :class:`~protocol.WebSocketCommonProtocol` instances can be used as +* :class:`~legacy.protocol.WebSocketCommonProtocol` instances can be used as asynchronous iterators on Python ≥ 3.6. They yield incoming messages. -* Added :func:`~server.unix_serve` for listening on Unix sockets. +* Added :func:`~legacy.server.unix_serve` for listening on Unix sockets. -* Added the :attr:`~server.WebSocketServer.sockets` attribute to the return - value of :func:`~server.serve`. +* Added the :attr:`~legacy.server.WebSocketServer.sockets` attribute to the return + value of :func:`~legacy.server.serve`. * Reorganized and extended documentation. @@ -384,15 +390,15 @@ Also: 3.4 ... -* Renamed :func:`~server.serve()` and :func:`~client.connect`'s ``klass`` +* Renamed :func:`~legacy.server.serve()` and :func:`~legacy.client.connect`'s ``klass`` argument to ``create_protocol`` to reflect that it can also be a callable. For backwards compatibility, ``klass`` is still supported. -* :func:`~server.serve` can be used as an asynchronous context manager on +* :func:`~legacy.server.serve` can be used as an asynchronous context manager on Python ≥ 3.5.1. * Added support for customizing handling of incoming connections with - :meth:`~server.WebSocketServerProtocol.process_request`. + :meth:`~legacy.server.WebSocketServerProtocol.process_request`. * Made read and write buffer sizes configurable. @@ -400,10 +406,10 @@ Also: * Added an optional C extension to speed up low-level operations. -* An invalid response status code during :func:`~client.connect` now raises +* An invalid response status code during :func:`~legacy.client.connect` now raises :class:`~exceptions.InvalidStatusCode` with a ``code`` attribute. -* Providing a ``sock`` argument to :func:`~client.connect` no longer +* Providing a ``sock`` argument to :func:`~legacy.client.connect` no longer crashes. 3.3 @@ -419,7 +425,7 @@ Also: ... * Added ``timeout``, ``max_size``, and ``max_queue`` arguments to - :func:`~client.connect()` and :func:`~server.serve`. + :func:`~legacy.client.connect()` and :func:`~legacy.server.serve`. * Made server shutdown more robust. @@ -436,11 +442,11 @@ Also: .. warning:: **Version 3.0 introduces a backwards-incompatible change in the** - :meth:`~protocol.WebSocketCommonProtocol.recv` **API.** + :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` **API.** **If you're upgrading from 2.x or earlier, please read this carefully.** - :meth:`~protocol.WebSocketCommonProtocol.recv` used to return ``None`` + :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` used to return ``None`` when the connection was closed. This required checking the return value of every call:: @@ -459,20 +465,20 @@ Also: In order to avoid stranding projects built upon an earlier version, the previous behavior can be restored by passing ``legacy_recv=True`` to - :func:`~server.serve`, :func:`~client.connect`, - :class:`~server.WebSocketServerProtocol`, or - :class:`~client.WebSocketClientProtocol`. ``legacy_recv`` isn't documented + :func:`~legacy.server.serve`, :func:`~legacy.client.connect`, + :class:`~legacy.server.WebSocketServerProtocol`, or + :class:`~legacy.client.WebSocketClientProtocol`. ``legacy_recv`` isn't documented in their signatures but isn't scheduled for deprecation either. Also: -* :func:`~client.connect` can be used as an asynchronous context manager on +* :func:`~legacy.client.connect` can be used as an asynchronous context manager on Python ≥ 3.5.1. * Updated documentation with ``await`` and ``async`` syntax from Python 3.5. -* :meth:`~protocol.WebSocketCommonProtocol.ping` and - :meth:`~protocol.WebSocketCommonProtocol.pong` support data passed as +* :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` and + :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` support data passed as :class:`str` in addition to :class:`bytes`. * Worked around an asyncio bug affecting connection termination under load. @@ -511,7 +517,7 @@ Also: * Returned a 403 status code instead of 400 when the request Origin isn't allowed. -* Canceling :meth:`~protocol.WebSocketCommonProtocol.recv` no longer drops +* Canceling :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` no longer drops the next message. * Clarified that the closing handshake can be initiated by the client. @@ -529,8 +535,8 @@ Also: * Supported non-default event loop. -* Added ``loop`` argument to :func:`~client.connect` and - :func:`~server.serve`. +* Added ``loop`` argument to :func:`~legacy.client.connect` and + :func:`~legacy.server.serve`. 2.3 ... @@ -557,9 +563,9 @@ Also: .. warning:: **Version 2.0 introduces a backwards-incompatible change in the** - :meth:`~protocol.WebSocketCommonProtocol.send`, - :meth:`~protocol.WebSocketCommonProtocol.ping`, and - :meth:`~protocol.WebSocketCommonProtocol.pong` **APIs.** + :meth:`~legacy.protocol.WebSocketCommonProtocol.send`, + :meth:`~legacy.protocol.WebSocketCommonProtocol.ping`, and + :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` **APIs.** **If you're upgrading from 1.x or earlier, please read this carefully.** diff --git a/docs/cheatsheet.rst b/docs/cheatsheet.rst index 4b95c9eea..a71f08d74 100644 --- a/docs/cheatsheet.rst +++ b/docs/cheatsheet.rst @@ -9,24 +9,24 @@ Server * Write a coroutine that handles a single connection. It receives a WebSocket protocol instance and the URI path in argument. - * Call :meth:`~protocol.WebSocketCommonProtocol.recv` and - :meth:`~protocol.WebSocketCommonProtocol.send` to receive and send + * Call :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` and + :meth:`~legacy.protocol.WebSocketCommonProtocol.send` to receive and send messages at any time. - * When :meth:`~protocol.WebSocketCommonProtocol.recv` or - :meth:`~protocol.WebSocketCommonProtocol.send` raises + * When :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` or + :meth:`~legacy.protocol.WebSocketCommonProtocol.send` raises :exc:`~exceptions.ConnectionClosed`, clean up and exit. If you started other :class:`asyncio.Task`, terminate them before exiting. - * If you aren't awaiting :meth:`~protocol.WebSocketCommonProtocol.recv`, - consider awaiting :meth:`~protocol.WebSocketCommonProtocol.wait_closed` + * If you aren't awaiting :meth:`~legacy.protocol.WebSocketCommonProtocol.recv`, + consider awaiting :meth:`~legacy.protocol.WebSocketCommonProtocol.wait_closed` to detect quickly when the connection is closed. - * You may :meth:`~protocol.WebSocketCommonProtocol.ping` or - :meth:`~protocol.WebSocketCommonProtocol.pong` if you wish but it isn't + * You may :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` or + :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` if you wish but it isn't needed in general. -* Create a server with :func:`~server.serve` which is similar to asyncio's +* Create a server with :func:`~legacy.server.serve` which is similar to asyncio's :meth:`~asyncio.AbstractEventLoop.create_server`. You can also use it as an asynchronous context manager. @@ -35,30 +35,30 @@ Server handler exits normally or with an exception. * For advanced customization, you may subclass - :class:`~server.WebSocketServerProtocol` and pass either this subclass or + :class:`~legacy.server.WebSocketServerProtocol` and pass either this subclass or a factory function as the ``create_protocol`` argument. Client ------ -* Create a client with :func:`~client.connect` which is similar to asyncio's +* Create a client with :func:`~legacy.client.connect` which is similar to asyncio's :meth:`~asyncio.BaseEventLoop.create_connection`. You can also use it as an asynchronous context manager. * For advanced customization, you may subclass - :class:`~server.WebSocketClientProtocol` and pass either this subclass or + :class:`~legacy.server.WebSocketClientProtocol` and pass either this subclass or a factory function as the ``create_protocol`` argument. -* Call :meth:`~protocol.WebSocketCommonProtocol.recv` and - :meth:`~protocol.WebSocketCommonProtocol.send` to receive and send messages +* Call :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` and + :meth:`~legacy.protocol.WebSocketCommonProtocol.send` to receive and send messages at any time. -* You may :meth:`~protocol.WebSocketCommonProtocol.ping` or - :meth:`~protocol.WebSocketCommonProtocol.pong` if you wish but it isn't +* You may :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` or + :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` if you wish but it isn't needed in general. -* If you aren't using :func:`~client.connect` as a context manager, call - :meth:`~protocol.WebSocketCommonProtocol.close` to terminate the connection. +* If you aren't using :func:`~legacy.client.connect` as a context manager, call + :meth:`~legacy.protocol.WebSocketCommonProtocol.close` to terminate the connection. .. _debugging: diff --git a/docs/deployment.rst b/docs/deployment.rst index 5b05afff1..ed025094d 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -24,7 +24,7 @@ Graceful shutdown You may want to close connections gracefully when shutting down the server, perhaps after executing some cleanup logic. There are two ways to achieve this -with the object returned by :func:`~server.serve`: +with the object returned by :func:`~legacy.server.serve`: - using it as a asynchronous context manager, or - calling its ``close()`` method, then waiting for its ``wait_closed()`` @@ -132,7 +132,7 @@ Under high load, if a server receives more messages than it can process, bufferbloat can result in excessive memory use. By default ``websockets`` has generous limits. It is strongly recommended to -adapt them to your application. When you call :func:`~server.serve`: +adapt them to your application. When you call :func:`~legacy.server.serve`: - Set ``max_size`` (default: 1 MiB, UTF-8 encoded) to the maximum size of messages your application generates. @@ -155,7 +155,7 @@ The author of ``websockets`` doesn't think that's a good idea, due to the widely different operational characteristics of HTTP and WebSocket. ``websockets`` provide minimal support for responding to HTTP requests with -the :meth:`~server.WebSocketServerProtocol.process_request` hook. Typical +the :meth:`~legacy.server.WebSocketServerProtocol.process_request` hook. Typical use cases include health checks. Here's an example: .. literalinclude:: ../example/health_check_server.py diff --git a/docs/design.rst b/docs/design.rst index 74279b87f..f2718370d 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -32,20 +32,20 @@ WebSocket connections go through a trivial state machine: Transitions happen in the following places: - ``CONNECTING -> OPEN``: in - :meth:`~protocol.WebSocketCommonProtocol.connection_open` which runs when + :meth:`~legacy.protocol.WebSocketCommonProtocol.connection_open` which runs when the :ref:`opening handshake ` completes and the WebSocket connection is established — not to be confused with :meth:`~asyncio.Protocol.connection_made` which runs when the TCP connection is established; - ``OPEN -> CLOSING``: in - :meth:`~protocol.WebSocketCommonProtocol.write_frame` immediately before + :meth:`~legacy.protocol.WebSocketCommonProtocol.write_frame` immediately before sending a close frame; since receiving a close frame triggers sending a close frame, this does the right thing regardless of which side started the :ref:`closing handshake `; also in - :meth:`~protocol.WebSocketCommonProtocol.fail_connection` which duplicates + :meth:`~legacy.protocol.WebSocketCommonProtocol.fail_connection` which duplicates a few lines of code from ``write_close_frame()`` and ``write_frame()``; - ``* -> CLOSED``: in - :meth:`~protocol.WebSocketCommonProtocol.connection_lost` which is always + :meth:`~legacy.protocol.WebSocketCommonProtocol.connection_lost` which is always called exactly once when the TCP connection is closed. Coroutines @@ -58,36 +58,36 @@ connection lifecycle on the client side. :target: _images/lifecycle.svg The lifecycle is identical on the server side, except inversion of control -makes the equivalent of :meth:`~client.connect` implicit. +makes the equivalent of :meth:`~legacy.client.connect` implicit. Coroutines shown in green are called by the application. Multiple coroutines may interact with the WebSocket connection concurrently. Coroutines shown in gray manage the connection. When the opening handshake -succeeds, :meth:`~protocol.WebSocketCommonProtocol.connection_open` starts +succeeds, :meth:`~legacy.protocol.WebSocketCommonProtocol.connection_open` starts two tasks: -- :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` runs - :meth:`~protocol.WebSocketCommonProtocol.transfer_data` which handles - incoming data and lets :meth:`~protocol.WebSocketCommonProtocol.recv` +- :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` runs + :meth:`~legacy.protocol.WebSocketCommonProtocol.transfer_data` which handles + incoming data and lets :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` consume it. It may be canceled to terminate the connection. It never exits with an exception other than :exc:`~asyncio.CancelledError`. See :ref:`data transfer ` below. -- :attr:`~protocol.WebSocketCommonProtocol.keepalive_ping_task` runs - :meth:`~protocol.WebSocketCommonProtocol.keepalive_ping` which sends Ping +- :attr:`~legacy.protocol.WebSocketCommonProtocol.keepalive_ping_task` runs + :meth:`~legacy.protocol.WebSocketCommonProtocol.keepalive_ping` which sends Ping frames at regular intervals and ensures that corresponding Pong frames are received. It is canceled when the connection terminates. It never exits with an exception other than :exc:`~asyncio.CancelledError`. -- :attr:`~protocol.WebSocketCommonProtocol.close_connection_task` runs - :meth:`~protocol.WebSocketCommonProtocol.close_connection` which waits for +- :attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` runs + :meth:`~legacy.protocol.WebSocketCommonProtocol.close_connection` which waits for the data transfer to terminate, then takes care of closing the TCP connection. It must not be canceled. It never exits with an exception. See :ref:`connection termination ` below. -Besides, :meth:`~protocol.WebSocketCommonProtocol.fail_connection` starts -the same :attr:`~protocol.WebSocketCommonProtocol.close_connection_task` when +Besides, :meth:`~legacy.protocol.WebSocketCommonProtocol.fail_connection` starts +the same :attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` when the opening handshake fails, in order to close the TCP connection. Splitting the responsibilities between two tasks makes it easier to guarantee @@ -99,11 +99,11 @@ that ``websockets`` can terminate connections: regardless of whether the connection terminates normally or abnormally. -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` completes when no +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` completes when no more data will be received on the connection. Under normal circumstances, it exits after exchanging close frames. -:attr:`~protocol.WebSocketCommonProtocol.close_connection_task` completes when +:attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` completes when the TCP connection is closed. @@ -113,7 +113,7 @@ Opening handshake ----------------- ``websockets`` performs the opening handshake when establishing a WebSocket -connection. On the client side, :meth:`~client.connect` executes it before +connection. On the client side, :meth:`~legacy.client.connect` executes it before returning the protocol to the caller. On the server side, it's executed before passing the protocol to the ``ws_handler`` coroutine handling the connection. @@ -122,26 +122,26 @@ request and the server replies with an HTTP Switching Protocols response — ``websockets`` aims at keeping the implementation of both sides consistent with one another. -On the client side, :meth:`~client.WebSocketClientProtocol.handshake`: +On the client side, :meth:`~legacy.client.WebSocketClientProtocol.handshake`: - builds a HTTP request based on the ``uri`` and parameters passed to - :meth:`~client.connect`; + :meth:`~legacy.client.connect`; - writes the HTTP request to the network; - reads a HTTP response from the network; - checks the HTTP response, validates ``extensions`` and ``subprotocol``, and configures the protocol accordingly; - moves to the ``OPEN`` state. -On the server side, :meth:`~server.WebSocketServerProtocol.handshake`: +On the server side, :meth:`~legacy.server.WebSocketServerProtocol.handshake`: - reads a HTTP request from the network; -- calls :meth:`~server.WebSocketServerProtocol.process_request` which may +- calls :meth:`~legacy.server.WebSocketServerProtocol.process_request` which may abort the WebSocket handshake and return a HTTP response instead; this hook only makes sense on the server side; - checks the HTTP request, negotiates ``extensions`` and ``subprotocol``, and configures the protocol accordingly; - builds a HTTP response based on the above and parameters passed to - :meth:`~server.serve`; + :meth:`~legacy.server.serve`; - writes the HTTP response to the network; - moves to the ``OPEN`` state; - returns the ``path`` part of the ``uri``. @@ -177,16 +177,16 @@ differences between a server and a client: These differences are so minor that all the logic for `data framing`_, for `sending and receiving data`_ and for `closing the connection`_ is implemented -in the same class, :class:`~protocol.WebSocketCommonProtocol`. +in the same class, :class:`~legacy.protocol.WebSocketCommonProtocol`. .. _data framing: https://tools.ietf.org/html/rfc6455#section-5 .. _sending and receiving data: https://tools.ietf.org/html/rfc6455#section-6 .. _closing the connection: https://tools.ietf.org/html/rfc6455#section-7 -The :attr:`~protocol.WebSocketCommonProtocol.is_client` attribute tells which +The :attr:`~legacy.protocol.WebSocketCommonProtocol.is_client` attribute tells which side a protocol instance is managing. This attribute is defined on the -:attr:`~server.WebSocketServerProtocol` and -:attr:`~client.WebSocketClientProtocol` classes. +:attr:`~legacy.server.WebSocketServerProtocol` and +:attr:`~legacy.client.WebSocketClientProtocol` classes. Data flow ......... @@ -210,11 +210,11 @@ The left side of the diagram shows how ``websockets`` receives data. Incoming data is written to a :class:`~asyncio.StreamReader` in order to implement flow control and provide backpressure on the TCP connection. -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task`, which is started +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task`, which is started when the WebSocket connection is established, processes this data. When it receives data frames, it reassembles fragments and puts the resulting -messages in the :attr:`~protocol.WebSocketCommonProtocol.messages` queue. +messages in the :attr:`~legacy.protocol.WebSocketCommonProtocol.messages` queue. When it encounters a control frame: @@ -226,11 +226,11 @@ When it encounters a control frame: Running this process in a task guarantees that control frames are processed promptly. Without such a task, ``websockets`` would depend on the application to drive the connection by having exactly one coroutine awaiting -:meth:`~protocol.WebSocketCommonProtocol.recv` at any time. While this +:meth:`~legacy.protocol.WebSocketCommonProtocol.recv` at any time. While this happens naturally in many use cases, it cannot be relied upon. -Then :meth:`~protocol.WebSocketCommonProtocol.recv` fetches the next message -from the :attr:`~protocol.WebSocketCommonProtocol.messages` queue, with some +Then :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` fetches the next message +from the :attr:`~legacy.protocol.WebSocketCommonProtocol.messages` queue, with some complexity added for handling backpressure and termination correctly. Sending data @@ -238,18 +238,18 @@ Sending data The right side of the diagram shows how ``websockets`` sends data. -:meth:`~protocol.WebSocketCommonProtocol.send` writes one or several data +:meth:`~legacy.protocol.WebSocketCommonProtocol.send` writes one or several data frames containing the message. While sending a fragmented message, concurrent -calls to :meth:`~protocol.WebSocketCommonProtocol.send` are put on hold until +calls to :meth:`~legacy.protocol.WebSocketCommonProtocol.send` are put on hold until all fragments are sent. This makes concurrent calls safe. -:meth:`~protocol.WebSocketCommonProtocol.ping` writes a ping frame and +:meth:`~legacy.protocol.WebSocketCommonProtocol.ping` writes a ping frame and yields a :class:`~asyncio.Future` which will be completed when a matching pong frame is received. -:meth:`~protocol.WebSocketCommonProtocol.pong` writes a pong frame. +:meth:`~legacy.protocol.WebSocketCommonProtocol.pong` writes a pong frame. -:meth:`~protocol.WebSocketCommonProtocol.close` writes a close frame and +:meth:`~legacy.protocol.WebSocketCommonProtocol.close` writes a close frame and waits for the TCP connection to terminate. Outgoing data is written to a :class:`~asyncio.StreamWriter` in order to @@ -261,17 +261,17 @@ Closing handshake ................. When the other side of the connection initiates the closing handshake, -:meth:`~protocol.WebSocketCommonProtocol.read_message` receives a close +:meth:`~legacy.protocol.WebSocketCommonProtocol.read_message` receives a close frame while in the ``OPEN`` state. It moves to the ``CLOSING`` state, sends a close frame, and returns ``None``, causing -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` to terminate. +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` to terminate. When this side of the connection initiates the closing handshake with -:meth:`~protocol.WebSocketCommonProtocol.close`, it moves to the ``CLOSING`` +:meth:`~legacy.protocol.WebSocketCommonProtocol.close`, it moves to the ``CLOSING`` state and sends a close frame. When the other side sends a close frame, -:meth:`~protocol.WebSocketCommonProtocol.read_message` receives it in the +:meth:`~legacy.protocol.WebSocketCommonProtocol.read_message` receives it in the ``CLOSING`` state and returns ``None``, also causing -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` to terminate. +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` to terminate. If the other side doesn't send a close frame within the connection's close timeout, ``websockets`` :ref:`fails the connection `. @@ -288,31 +288,31 @@ Then ``websockets`` terminates the TCP connection. Connection termination ---------------------- -:attr:`~protocol.WebSocketCommonProtocol.close_connection_task`, which is +:attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task`, which is started when the WebSocket connection is established, is responsible for eventually closing the TCP connection. -First :attr:`~protocol.WebSocketCommonProtocol.close_connection_task` waits -for :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` to terminate, +First :attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` waits +for :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` to terminate, which may happen as a result of: - a successful closing handshake: as explained above, this exits the infinite - loop in :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task`; + loop in :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task`; - a timeout while waiting for the closing handshake to complete: this cancels - :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task`; + :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task`; - a protocol error, including connection errors: depending on the exception, - :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` :ref:`fails the + :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` :ref:`fails the connection ` with a suitable code and exits. -:attr:`~protocol.WebSocketCommonProtocol.close_connection_task` is separate -from :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` to make it +:attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` is separate +from :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` to make it easier to implement the timeout on the closing handshake. Canceling -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` creates no risk -of canceling :attr:`~protocol.WebSocketCommonProtocol.close_connection_task` +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` creates no risk +of canceling :attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` and failing to close the TCP connection, thus leaking resources. -Then :attr:`~protocol.WebSocketCommonProtocol.close_connection_task` cancels -:attr:`~protocol.WebSocketCommonProtocol.keepalive_ping`. This task has no +Then :attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` cancels +:attr:`~legacy.protocol.WebSocketCommonProtocol.keepalive_ping`. This task has no protocol compliance responsibilities. Terminating it to avoid leaking it is the only concern. @@ -334,11 +334,11 @@ If the opening handshake doesn't complete successfully, ``websockets`` fails the connection by closing the TCP connection. Once the opening handshake has completed, ``websockets`` fails the connection -by canceling :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` and +by canceling :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` and sending a close frame if appropriate. -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` exits, unblocking -:attr:`~protocol.WebSocketCommonProtocol.close_connection_task`, which closes +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` exits, unblocking +:attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task`, which closes the TCP connection. @@ -414,45 +414,45 @@ happen on the client side. On the server side, the opening handshake is managed by ``websockets`` and nothing results in a cancellation. Once the WebSocket connection is established, internal tasks -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` and -:attr:`~protocol.WebSocketCommonProtocol.close_connection_task` mustn't get +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` and +:attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` mustn't get accidentally canceled if a coroutine that awaits them is canceled. In other words, they must be shielded from cancellation. -:meth:`~protocol.WebSocketCommonProtocol.recv` waits for the next message in -the queue or for :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` +:meth:`~legacy.protocol.WebSocketCommonProtocol.recv` waits for the next message in +the queue or for :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` to terminate, whichever comes first. It relies on :func:`~asyncio.wait` for waiting on two futures in parallel. As a consequence, even though it's waiting on a :class:`~asyncio.Future` signaling the next message and on -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task`, it doesn't +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task`, it doesn't propagate cancellation to them. -:meth:`~protocol.WebSocketCommonProtocol.ensure_open` is called by -:meth:`~protocol.WebSocketCommonProtocol.send`, -:meth:`~protocol.WebSocketCommonProtocol.ping`, and -:meth:`~protocol.WebSocketCommonProtocol.pong`. When the connection state is +:meth:`~legacy.protocol.WebSocketCommonProtocol.ensure_open` is called by +:meth:`~legacy.protocol.WebSocketCommonProtocol.send`, +:meth:`~legacy.protocol.WebSocketCommonProtocol.ping`, and +:meth:`~legacy.protocol.WebSocketCommonProtocol.pong`. When the connection state is ``CLOSING``, it waits for -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` but shields it to +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` but shields it to prevent cancellation. -:meth:`~protocol.WebSocketCommonProtocol.close` waits for the data transfer +:meth:`~legacy.protocol.WebSocketCommonProtocol.close` waits for the data transfer task to terminate with :func:`~asyncio.wait_for`. If it's canceled or if the -timeout elapses, :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` +timeout elapses, :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` is canceled, which is correct at this point. -:meth:`~protocol.WebSocketCommonProtocol.close` then waits for -:attr:`~protocol.WebSocketCommonProtocol.close_connection_task` but shields it +:meth:`~legacy.protocol.WebSocketCommonProtocol.close` then waits for +:attr:`~legacy.protocol.WebSocketCommonProtocol.close_connection_task` but shields it to prevent cancellation. -:meth:`~protocol.WebSocketCommonProtocol.close` and -:func:`~protocol.WebSocketCommonProtocol.fail_connection` are the only -places where :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` may +:meth:`~legacy.protocol.WebSocketCommonProtocol.close` and +:func:`~legacy.protocol.WebSocketCommonProtocol.fail_connection` are the only +places where :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` may be canceled. -:attr:`~protocol.WebSocketCommonProtocol.close_connnection_task` starts by -waiting for :attr:`~protocol.WebSocketCommonProtocol.transfer_data_task`. It +:attr:`~legacy.protocol.WebSocketCommonProtocol.close_connnection_task` starts by +waiting for :attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task`. It catches :exc:`~asyncio.CancelledError` to prevent a cancellation of -:attr:`~protocol.WebSocketCommonProtocol.transfer_data_task` from propagating -to :attr:`~protocol.WebSocketCommonProtocol.close_connnection_task`. +:attr:`~legacy.protocol.WebSocketCommonProtocol.transfer_data_task` from propagating +to :attr:`~legacy.protocol.WebSocketCommonProtocol.close_connnection_task`. .. _backpressure: @@ -519,47 +519,47 @@ For each connection, the receiving side contains these buffers: - OS buffers: tuning them is an advanced optimization. - :class:`~asyncio.StreamReader` bytes buffer: the default limit is 64 KiB. You can set another limit by passing a ``read_limit`` keyword argument to - :func:`~client.connect()` or :func:`~server.serve`. + :func:`~legacy.client.connect()` or :func:`~legacy.server.serve`. - Incoming messages :class:`~collections.deque`: its size depends both on the size and the number of messages it contains. By default the maximum UTF-8 encoded size is 1 MiB and the maximum number is 32. In the worst case, after UTF-8 decoding, a single message could take up to 4 MiB of memory and the overall memory consumption could reach 128 MiB. You should adjust these limits by setting the ``max_size`` and ``max_queue`` keyword arguments of - :func:`~client.connect()` or :func:`~server.serve` according to your + :func:`~legacy.client.connect()` or :func:`~legacy.server.serve` according to your application's requirements. For each connection, the sending side contains these buffers: - :class:`~asyncio.StreamWriter` bytes buffer: the default size is 64 KiB. You can set another limit by passing a ``write_limit`` keyword argument to - :func:`~client.connect()` or :func:`~server.serve`. + :func:`~legacy.client.connect()` or :func:`~legacy.server.serve`. - OS buffers: tuning them is an advanced optimization. Concurrency ----------- -Awaiting any combination of :meth:`~protocol.WebSocketCommonProtocol.recv`, -:meth:`~protocol.WebSocketCommonProtocol.send`, -:meth:`~protocol.WebSocketCommonProtocol.close` -:meth:`~protocol.WebSocketCommonProtocol.ping`, or -:meth:`~protocol.WebSocketCommonProtocol.pong` concurrently is safe, including +Awaiting any combination of :meth:`~legacy.protocol.WebSocketCommonProtocol.recv`, +:meth:`~legacy.protocol.WebSocketCommonProtocol.send`, +:meth:`~legacy.protocol.WebSocketCommonProtocol.close` +:meth:`~legacy.protocol.WebSocketCommonProtocol.ping`, or +:meth:`~legacy.protocol.WebSocketCommonProtocol.pong` concurrently is safe, including multiple calls to the same method, with one exception and one limitation. * **Only one coroutine can receive messages at a time.** This constraint avoids non-deterministic behavior (and simplifies the implementation). If a - coroutine is awaiting :meth:`~protocol.WebSocketCommonProtocol.recv`, + coroutine is awaiting :meth:`~legacy.protocol.WebSocketCommonProtocol.recv`, awaiting it again in another coroutine raises :exc:`RuntimeError`. * **Sending a fragmented message forces serialization.** Indeed, the WebSocket protocol doesn't support multiplexing messages. If a coroutine is awaiting - :meth:`~protocol.WebSocketCommonProtocol.send` to send a fragmented message, + :meth:`~legacy.protocol.WebSocketCommonProtocol.send` to send a fragmented message, awaiting it again in another coroutine waits until the first call completes. This will be transparent in many cases. It may be a concern if the fragmented message is generated slowly by an asynchronous iterator. Receiving frames is independent from sending frames. This isolates -:meth:`~protocol.WebSocketCommonProtocol.recv`, which receives frames, from +:meth:`~legacy.protocol.WebSocketCommonProtocol.recv`, which receives frames, from the other methods, which send frames. While the connection is open, each frame is sent with a single write. Combined diff --git a/docs/extensions.rst b/docs/extensions.rst index 400034090..dea91219e 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -14,8 +14,9 @@ Per-Message Deflate, specified in :rfc:`7692`. Per-Message Deflate ------------------- -:func:`~server.serve()` and :func:`~client.connect` enable the Per-Message -Deflate extension by default. You can disable this with ``compression=None``. +:func:`~legacy.server.serve()` and :func:`~legacy.client.connect` enable the +Per-Message Deflate extension by default. You can disable this with +``compression=None``. You can also configure the Per-Message Deflate extension explicitly if you want to customize its parameters. diff --git a/docs/faq.rst b/docs/faq.rst index 4a083e2d0..eee14dda8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -80,13 +80,13 @@ How do I get access HTTP headers, for example cookies? ...................................................... To access HTTP headers during the WebSocket handshake, you can override -:attr:`~server.WebSocketServerProtocol.process_request`:: +:attr:`~legacy.server.WebSocketServerProtocol.process_request`:: async def process_request(self, path, request_headers): cookies = request_header["Cookie"] Once the connection is established, they're available in -:attr:`~protocol.WebSocketServerProtocol.request_headers`:: +:attr:`~legacy.protocol.WebSocketServerProtocol.request_headers`:: async def handler(websocket, path): cookies = websocket.request_headers["Cookie"] @@ -94,7 +94,7 @@ Once the connection is established, they're available in How do I get the IP address of the client connecting to my server? .................................................................. -It's available in :attr:`~protocol.WebSocketCommonProtocol.remote_address`:: +It's available in :attr:`~legacy.protocol.WebSocketCommonProtocol.remote_address`:: async def handler(websocket, path): remote_ip = websocket.remote_address[0] @@ -121,7 +121,7 @@ Providing a HTTP server is out of scope for websockets. It only aims at providing a WebSocket server. There's limited support for returning HTTP responses with the -:attr:`~server.WebSocketServerProtocol.process_request` hook. +:attr:`~legacy.server.WebSocketServerProtocol.process_request` hook. If you need more, pick a HTTP server and run it separately. Client side diff --git a/docs/intro.rst b/docs/intro.rst index 8aaaeddca..c77139cab 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -143,7 +143,7 @@ For getting messages from a ``producer`` coroutine and sending them:: In this example, ``producer`` represents your business logic for generating messages to send on the WebSocket connection. -:meth:`~protocol.WebSocketCommonProtocol.send` raises a +:meth:`~legacy.protocol.WebSocketCommonProtocol.send` raises a :exc:`~exceptions.ConnectionClosed` exception when the client disconnects, which breaks out of the ``while True`` loop. diff --git a/setup.py b/setup.py index f35819247..85d899cb4 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ if sys.version_info[:3] < (3, 6, 1): raise Exception("websockets requires Python >= 3.6.1.") -packages = ['websockets', 'websockets/extensions'] +packages = ['websockets', 'websockets/legacy', 'websockets/extensions'] ext_modules = [ setuptools.Extension( diff --git a/src/websockets/__init__.py b/src/websockets/__init__.py index c4accaca1..0242e7942 100644 --- a/src/websockets/__init__.py +++ b/src/websockets/__init__.py @@ -1,11 +1,13 @@ # This relies on each of the submodules having an __all__ variable. -from .auth import * # noqa -from .client import * # noqa +from .client import * from .datastructures import * # noqa from .exceptions import * # noqa -from .protocol import * # noqa -from .server import * # noqa +from .legacy.auth import * # noqa +from .legacy.client import * # noqa +from .legacy.protocol import * # noqa +from .legacy.server import * # noqa +from .server import * from .typing import * # noqa from .uri import * # noqa from .version import version as __version__ # noqa diff --git a/src/websockets/__main__.py b/src/websockets/__main__.py index bce3e4bbb..d44e34e74 100644 --- a/src/websockets/__main__.py +++ b/src/websockets/__main__.py @@ -6,8 +6,8 @@ import threading from typing import Any, Set -from .client import connect from .exceptions import ConnectionClosed, format_close +from .legacy.client import connect if sys.platform == "win32": diff --git a/src/websockets/auth.py b/src/websockets/auth.py index c1b7a0b1a..c8839c401 100644 --- a/src/websockets/auth.py +++ b/src/websockets/auth.py @@ -1,165 +1,4 @@ -""" -:mod:`websockets.auth` provides HTTP Basic Authentication according to -:rfc:`7235` and :rfc:`7617`. - -""" - - -import functools -import http -from typing import Any, Awaitable, Callable, Iterable, Optional, Tuple, Union, cast - -from .asyncio_server import HTTPResponse, WebSocketServerProtocol -from .datastructures import Headers -from .exceptions import InvalidHeader -from .headers import build_www_authenticate_basic, parse_authorization_basic +from .legacy.auth import BasicAuthWebSocketServerProtocol, basic_auth_protocol_factory __all__ = ["BasicAuthWebSocketServerProtocol", "basic_auth_protocol_factory"] - -Credentials = Tuple[str, str] - - -def is_credentials(value: Any) -> bool: - try: - username, password = value - except (TypeError, ValueError): - return False - else: - return isinstance(username, str) and isinstance(password, str) - - -class BasicAuthWebSocketServerProtocol(WebSocketServerProtocol): - """ - WebSocket server protocol that enforces HTTP Basic Auth. - - """ - - def __init__( - self, - *args: Any, - realm: str, - check_credentials: Callable[[str, str], Awaitable[bool]], - **kwargs: Any, - ) -> None: - self.realm = realm - self.check_credentials = check_credentials - super().__init__(*args, **kwargs) - - async def process_request( - self, path: str, request_headers: Headers - ) -> Optional[HTTPResponse]: - """ - Check HTTP Basic Auth and return a HTTP 401 or 403 response if needed. - - If authentication succeeds, the username of the authenticated user is - stored in the ``username`` attribute. - - """ - try: - authorization = request_headers["Authorization"] - except KeyError: - return ( - http.HTTPStatus.UNAUTHORIZED, - [("WWW-Authenticate", build_www_authenticate_basic(self.realm))], - b"Missing credentials\n", - ) - - try: - username, password = parse_authorization_basic(authorization) - except InvalidHeader: - return ( - http.HTTPStatus.UNAUTHORIZED, - [("WWW-Authenticate", build_www_authenticate_basic(self.realm))], - b"Unsupported credentials\n", - ) - - if not await self.check_credentials(username, password): - return ( - http.HTTPStatus.UNAUTHORIZED, - [("WWW-Authenticate", build_www_authenticate_basic(self.realm))], - b"Invalid credentials\n", - ) - - self.username = username - - return await super().process_request(path, request_headers) - - -def basic_auth_protocol_factory( - realm: str, - credentials: Optional[Union[Credentials, Iterable[Credentials]]] = None, - check_credentials: Optional[Callable[[str, str], Awaitable[bool]]] = None, - create_protocol: Optional[Callable[[Any], BasicAuthWebSocketServerProtocol]] = None, -) -> Callable[[Any], BasicAuthWebSocketServerProtocol]: - """ - Protocol factory that enforces HTTP Basic Auth. - - ``basic_auth_protocol_factory`` is designed to integrate with - :func:`~websockets.server.serve` like this:: - - websockets.serve( - ..., - create_protocol=websockets.basic_auth_protocol_factory( - realm="my dev server", - credentials=("hello", "iloveyou"), - ) - ) - - ``realm`` indicates the scope of protection. It should contain only ASCII - characters because the encoding of non-ASCII characters is undefined. - Refer to section 2.2 of :rfc:`7235` for details. - - ``credentials`` defines hard coded authorized credentials. It can be a - ``(username, password)`` pair or a list of such pairs. - - ``check_credentials`` defines a coroutine that checks whether credentials - are authorized. This coroutine receives ``username`` and ``password`` - arguments and returns a :class:`bool`. - - One of ``credentials`` or ``check_credentials`` must be provided but not - both. - - By default, ``basic_auth_protocol_factory`` creates a factory for building - :class:`BasicAuthWebSocketServerProtocol` instances. You can override this - with the ``create_protocol`` parameter. - - :param realm: scope of protection - :param credentials: hard coded credentials - :param check_credentials: coroutine that verifies credentials - :raises TypeError: if the credentials argument has the wrong type - - """ - if (credentials is None) == (check_credentials is None): - raise TypeError("provide either credentials or check_credentials") - - if credentials is not None: - if is_credentials(credentials): - - async def check_credentials(username: str, password: str) -> bool: - return (username, password) == credentials - - elif isinstance(credentials, Iterable): - credentials_list = list(credentials) - if all(is_credentials(item) for item in credentials_list): - credentials_dict = dict(credentials_list) - - async def check_credentials(username: str, password: str) -> bool: - return credentials_dict.get(username) == password - - else: - raise TypeError(f"invalid credentials argument: {credentials}") - - else: - raise TypeError(f"invalid credentials argument: {credentials}") - - if create_protocol is None: - # Not sure why mypy cannot figure this out. - create_protocol = cast( - Callable[[Any], BasicAuthWebSocketServerProtocol], - BasicAuthWebSocketServerProtocol, - ) - - return functools.partial( - create_protocol, realm=realm, check_credentials=check_credentials - ) diff --git a/src/websockets/client.py b/src/websockets/client.py index b7e407a45..8cababed5 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -2,7 +2,6 @@ import logging from typing import Generator, List, Optional, Sequence -from .asyncio_client import WebSocketClientProtocol, connect, unix_connect from .connection import CLIENT, CONNECTING, OPEN, Connection from .datastructures import Headers, HeadersLike, MultipleValuesError from .exceptions import ( @@ -25,6 +24,7 @@ ) from .http import USER_AGENT, build_host from .http11 import Request, Response +from .legacy.client import WebSocketClientProtocol, connect, unix_connect # noqa from .typing import ( ConnectionOption, ExtensionHeader, @@ -36,12 +36,7 @@ from .utils import accept_key, generate_key -__all__ = [ - "connect", - "unix_connect", - "ClientConnection", - "WebSocketClientProtocol", -] +__all__ = ["ClientConnection"] logger = logging.getLogger(__name__) @@ -64,7 +59,7 @@ def __init__( self.extra_headers = extra_headers self.key = generate_key() - def connect(self) -> Request: + def connect(self) -> Request: # noqa: F811 """ Create a WebSocket handshake request event to send to the server. diff --git a/src/websockets/exceptions.py b/src/websockets/exceptions.py index bdadae05e..e0860c743 100644 --- a/src/websockets/exceptions.py +++ b/src/websockets/exceptions.py @@ -301,7 +301,7 @@ class AbortHandshake(InvalidHandshake): This exception is an implementation detail. - The public API is :meth:`~server.WebSocketServerProtocol.process_request`. + The public API is :meth:`~legacy.server.WebSocketServerProtocol.process_request`. """ diff --git a/src/websockets/framing.py b/src/websockets/framing.py index b2996d788..2dadb5610 100644 --- a/src/websockets/framing.py +++ b/src/websockets/framing.py @@ -1,139 +1,6 @@ -""" -:mod:`websockets.framing` reads and writes WebSocket frames. - -It deals with a single frame at a time. Anything that depends on the sequence -of frames is implemented in :mod:`websockets.protocol`. - -See `section 5 of RFC 6455`_. - -.. _section 5 of RFC 6455: http://tools.ietf.org/html/rfc6455#section-5 - -""" - -import struct import warnings -from typing import Any, Awaitable, Callable, Optional, Sequence - -from .exceptions import PayloadTooBig, ProtocolError -from .frames import Frame as NewFrame, Opcode - -try: - from .speedups import apply_mask -except ImportError: # pragma: no cover - from .utils import apply_mask +from .legacy.framing import * # noqa warnings.warn("websockets.framing is deprecated", DeprecationWarning) - - -class Frame(NewFrame): - @classmethod - async def read( - cls, - reader: Callable[[int], Awaitable[bytes]], - *, - mask: bool, - max_size: Optional[int] = None, - extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, - ) -> "Frame": - """ - Read a WebSocket frame. - - :param reader: coroutine that reads exactly the requested number of - bytes, unless the end of file is reached - :param mask: whether the frame should be masked i.e. whether the read - happens on the server side - :param max_size: maximum payload size in bytes - :param extensions: list of classes with a ``decode()`` method that - transforms the frame and return a new frame; extensions are applied - in reverse order - :raises ~websockets.exceptions.PayloadTooBig: if the frame exceeds - ``max_size`` - :raises ~websockets.exceptions.ProtocolError: if the frame - contains incorrect values - - """ - - # Read the header. - data = await reader(2) - head1, head2 = struct.unpack("!BB", data) - - # While not Pythonic, this is marginally faster than calling bool(). - fin = True if head1 & 0b10000000 else False - rsv1 = True if head1 & 0b01000000 else False - rsv2 = True if head1 & 0b00100000 else False - rsv3 = True if head1 & 0b00010000 else False - - try: - opcode = Opcode(head1 & 0b00001111) - except ValueError as exc: - raise ProtocolError("invalid opcode") from exc - - if (True if head2 & 0b10000000 else False) != mask: - raise ProtocolError("incorrect masking") - - length = head2 & 0b01111111 - if length == 126: - data = await reader(2) - (length,) = struct.unpack("!H", data) - elif length == 127: - data = await reader(8) - (length,) = struct.unpack("!Q", data) - if max_size is not None and length > max_size: - raise PayloadTooBig(f"over size limit ({length} > {max_size} bytes)") - if mask: - mask_bits = await reader(4) - - # Read the data. - data = await reader(length) - if mask: - data = apply_mask(data, mask_bits) - - frame = cls(fin, opcode, data, rsv1, rsv2, rsv3) - - if extensions is None: - extensions = [] - for extension in reversed(extensions): - frame = cls(*extension.decode(frame, max_size=max_size)) - - frame.check() - - return frame - - def write( - self, - write: Callable[[bytes], Any], - *, - mask: bool, - extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, - ) -> None: - """ - Write a WebSocket frame. - - :param frame: frame to write - :param write: function that writes bytes - :param mask: whether the frame should be masked i.e. whether the write - happens on the client side - :param extensions: list of classes with an ``encode()`` method that - transform the frame and return a new frame; extensions are applied - in order - :raises ~websockets.exceptions.ProtocolError: if the frame - contains incorrect values - - """ - # The frame is written in a single call to write in order to prevent - # TCP fragmentation. See #68 for details. This also makes it safe to - # send frames concurrently from multiple coroutines. - write(self.serialize(mask=mask, extensions=extensions)) - - -# Backwards compatibility with previously documented public APIs -from .frames import parse_close # isort:skip # noqa -from .frames import prepare_ctrl as encode_data # isort:skip # noqa -from .frames import prepare_data # isort:skip # noqa -from .frames import serialize_close # isort:skip # noqa - - -# at the bottom to allow circular import, because Extension depends on Frame -import websockets.extensions.base # isort:skip # noqa diff --git a/src/websockets/handshake.py b/src/websockets/handshake.py index 3ff6c005d..cc4010d41 100644 --- a/src/websockets/handshake.py +++ b/src/websockets/handshake.py @@ -13,7 +13,7 @@ def build_request(headers: Headers) -> str: # pragma: no cover warnings.warn( "websockets.handshake.build_request is deprecated", DeprecationWarning ) - from .handshake_legacy import build_request + from .legacy.handshake import build_request return build_request(headers) @@ -22,7 +22,7 @@ def check_request(headers: Headers) -> str: # pragma: no cover warnings.warn( "websockets.handshake.check_request is deprecated", DeprecationWarning ) - from .handshake_legacy import check_request + from .legacy.handshake import check_request return check_request(headers) @@ -31,7 +31,7 @@ def build_response(headers: Headers, key: str) -> None: # pragma: no cover warnings.warn( "websockets.handshake.build_response is deprecated", DeprecationWarning ) - from .handshake_legacy import build_response + from .legacy.handshake import build_response return build_response(headers, key) @@ -40,6 +40,6 @@ def check_response(headers: Headers, key: str) -> None: # pragma: no cover warnings.warn( "websockets.handshake.check_response is deprecated", DeprecationWarning ) - from .handshake_legacy import check_response + from .legacy.handshake import check_response return check_response(headers, key) diff --git a/src/websockets/http.py b/src/websockets/http.py index ed3fe48d0..b05b78455 100644 --- a/src/websockets/http.py +++ b/src/websockets/http.py @@ -47,7 +47,7 @@ async def read_request( stream: asyncio.StreamReader, ) -> Tuple[str, Headers]: # pragma: no cover warnings.warn("websockets.http.read_request is deprecated", DeprecationWarning) - from .http_legacy import read_request + from .legacy.http import read_request return await read_request(stream) @@ -56,6 +56,6 @@ async def read_response( stream: asyncio.StreamReader, ) -> Tuple[int, str, Headers]: # pragma: no cover warnings.warn("websockets.http.read_response is deprecated", DeprecationWarning) - from .http_legacy import read_response + from .legacy.http import read_response return await read_response(stream) diff --git a/src/websockets/legacy/__init__.py b/src/websockets/legacy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/websockets/legacy/auth.py b/src/websockets/legacy/auth.py new file mode 100644 index 000000000..8cb60429a --- /dev/null +++ b/src/websockets/legacy/auth.py @@ -0,0 +1,165 @@ +""" +:mod:`websockets.legacy.auth` provides HTTP Basic Authentication according to +:rfc:`7235` and :rfc:`7617`. + +""" + + +import functools +import http +from typing import Any, Awaitable, Callable, Iterable, Optional, Tuple, Union, cast + +from ..datastructures import Headers +from ..exceptions import InvalidHeader +from ..headers import build_www_authenticate_basic, parse_authorization_basic +from .server import HTTPResponse, WebSocketServerProtocol + + +__all__ = ["BasicAuthWebSocketServerProtocol", "basic_auth_protocol_factory"] + +Credentials = Tuple[str, str] + + +def is_credentials(value: Any) -> bool: + try: + username, password = value + except (TypeError, ValueError): + return False + else: + return isinstance(username, str) and isinstance(password, str) + + +class BasicAuthWebSocketServerProtocol(WebSocketServerProtocol): + """ + WebSocket server protocol that enforces HTTP Basic Auth. + + """ + + def __init__( + self, + *args: Any, + realm: str, + check_credentials: Callable[[str, str], Awaitable[bool]], + **kwargs: Any, + ) -> None: + self.realm = realm + self.check_credentials = check_credentials + super().__init__(*args, **kwargs) + + async def process_request( + self, path: str, request_headers: Headers + ) -> Optional[HTTPResponse]: + """ + Check HTTP Basic Auth and return a HTTP 401 or 403 response if needed. + + If authentication succeeds, the username of the authenticated user is + stored in the ``username`` attribute. + + """ + try: + authorization = request_headers["Authorization"] + except KeyError: + return ( + http.HTTPStatus.UNAUTHORIZED, + [("WWW-Authenticate", build_www_authenticate_basic(self.realm))], + b"Missing credentials\n", + ) + + try: + username, password = parse_authorization_basic(authorization) + except InvalidHeader: + return ( + http.HTTPStatus.UNAUTHORIZED, + [("WWW-Authenticate", build_www_authenticate_basic(self.realm))], + b"Unsupported credentials\n", + ) + + if not await self.check_credentials(username, password): + return ( + http.HTTPStatus.UNAUTHORIZED, + [("WWW-Authenticate", build_www_authenticate_basic(self.realm))], + b"Invalid credentials\n", + ) + + self.username = username + + return await super().process_request(path, request_headers) + + +def basic_auth_protocol_factory( + realm: str, + credentials: Optional[Union[Credentials, Iterable[Credentials]]] = None, + check_credentials: Optional[Callable[[str, str], Awaitable[bool]]] = None, + create_protocol: Optional[Callable[[Any], BasicAuthWebSocketServerProtocol]] = None, +) -> Callable[[Any], BasicAuthWebSocketServerProtocol]: + """ + Protocol factory that enforces HTTP Basic Auth. + + ``basic_auth_protocol_factory`` is designed to integrate with + :func:`~websockets.legacy.server.serve` like this:: + + websockets.serve( + ..., + create_protocol=websockets.basic_auth_protocol_factory( + realm="my dev server", + credentials=("hello", "iloveyou"), + ) + ) + + ``realm`` indicates the scope of protection. It should contain only ASCII + characters because the encoding of non-ASCII characters is undefined. + Refer to section 2.2 of :rfc:`7235` for details. + + ``credentials`` defines hard coded authorized credentials. It can be a + ``(username, password)`` pair or a list of such pairs. + + ``check_credentials`` defines a coroutine that checks whether credentials + are authorized. This coroutine receives ``username`` and ``password`` + arguments and returns a :class:`bool`. + + One of ``credentials`` or ``check_credentials`` must be provided but not + both. + + By default, ``basic_auth_protocol_factory`` creates a factory for building + :class:`BasicAuthWebSocketServerProtocol` instances. You can override this + with the ``create_protocol`` parameter. + + :param realm: scope of protection + :param credentials: hard coded credentials + :param check_credentials: coroutine that verifies credentials + :raises TypeError: if the credentials argument has the wrong type + + """ + if (credentials is None) == (check_credentials is None): + raise TypeError("provide either credentials or check_credentials") + + if credentials is not None: + if is_credentials(credentials): + + async def check_credentials(username: str, password: str) -> bool: + return (username, password) == credentials + + elif isinstance(credentials, Iterable): + credentials_list = list(credentials) + if all(is_credentials(item) for item in credentials_list): + credentials_dict = dict(credentials_list) + + async def check_credentials(username: str, password: str) -> bool: + return credentials_dict.get(username) == password + + else: + raise TypeError(f"invalid credentials argument: {credentials}") + + else: + raise TypeError(f"invalid credentials argument: {credentials}") + + if create_protocol is None: + # Not sure why mypy cannot figure this out. + create_protocol = cast( + Callable[[Any], BasicAuthWebSocketServerProtocol], + BasicAuthWebSocketServerProtocol, + ) + + return functools.partial( + create_protocol, realm=realm, check_credentials=check_credentials + ) diff --git a/src/websockets/asyncio_client.py b/src/websockets/legacy/client.py similarity index 97% rename from src/websockets/asyncio_client.py rename to src/websockets/legacy/client.py index 3f406170a..27f6e8209 100644 --- a/src/websockets/asyncio_client.py +++ b/src/websockets/legacy/client.py @@ -1,5 +1,5 @@ """ -:mod:`websockets.client` defines the WebSocket client APIs. +:mod:`websockets.legacy.client` defines the WebSocket client APIs. """ @@ -11,8 +11,8 @@ from types import TracebackType from typing import Any, Callable, Generator, List, Optional, Sequence, Tuple, Type, cast -from .datastructures import Headers, HeadersLike -from .exceptions import ( +from ..datastructures import Headers, HeadersLike +from ..exceptions import ( InvalidHandshake, InvalidHeader, InvalidMessage, @@ -21,21 +21,21 @@ RedirectHandshake, SecurityError, ) -from .extensions.base import ClientExtensionFactory, Extension -from .extensions.permessage_deflate import enable_client_permessage_deflate -from .handshake_legacy import build_request, check_response -from .headers import ( +from ..extensions.base import ClientExtensionFactory, Extension +from ..extensions.permessage_deflate import enable_client_permessage_deflate +from ..headers import ( build_authorization_basic, build_extension, build_subprotocol, parse_extension, parse_subprotocol, ) -from .http import USER_AGENT, build_host -from .http_legacy import read_response +from ..http import USER_AGENT, build_host +from ..typing import ExtensionHeader, Origin, Subprotocol +from ..uri import WebSocketURI, parse_uri +from .handshake import build_request, check_response +from .http import read_response from .protocol import WebSocketCommonProtocol -from .typing import ExtensionHeader, Origin, Subprotocol -from .uri import WebSocketURI, parse_uri __all__ = ["connect", "unix_connect", "WebSocketClientProtocol"] diff --git a/src/websockets/legacy/framing.py b/src/websockets/legacy/framing.py new file mode 100644 index 000000000..e41c295dd --- /dev/null +++ b/src/websockets/legacy/framing.py @@ -0,0 +1,135 @@ +""" +:mod:`websockets.legacy.framing` reads and writes WebSocket frames. + +It deals with a single frame at a time. Anything that depends on the sequence +of frames is implemented in :mod:`websockets.legacy.protocol`. + +See `section 5 of RFC 6455`_. + +.. _section 5 of RFC 6455: http://tools.ietf.org/html/rfc6455#section-5 + +""" + +import struct +from typing import Any, Awaitable, Callable, Optional, Sequence + +from ..exceptions import PayloadTooBig, ProtocolError +from ..frames import Frame as NewFrame, Opcode + + +try: + from ..speedups import apply_mask +except ImportError: # pragma: no cover + from ..utils import apply_mask + + +class Frame(NewFrame): + @classmethod + async def read( + cls, + reader: Callable[[int], Awaitable[bytes]], + *, + mask: bool, + max_size: Optional[int] = None, + extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + ) -> "Frame": + """ + Read a WebSocket frame. + + :param reader: coroutine that reads exactly the requested number of + bytes, unless the end of file is reached + :param mask: whether the frame should be masked i.e. whether the read + happens on the server side + :param max_size: maximum payload size in bytes + :param extensions: list of classes with a ``decode()`` method that + transforms the frame and return a new frame; extensions are applied + in reverse order + :raises ~websockets.exceptions.PayloadTooBig: if the frame exceeds + ``max_size`` + :raises ~websockets.exceptions.ProtocolError: if the frame + contains incorrect values + + """ + + # Read the header. + data = await reader(2) + head1, head2 = struct.unpack("!BB", data) + + # While not Pythonic, this is marginally faster than calling bool(). + fin = True if head1 & 0b10000000 else False + rsv1 = True if head1 & 0b01000000 else False + rsv2 = True if head1 & 0b00100000 else False + rsv3 = True if head1 & 0b00010000 else False + + try: + opcode = Opcode(head1 & 0b00001111) + except ValueError as exc: + raise ProtocolError("invalid opcode") from exc + + if (True if head2 & 0b10000000 else False) != mask: + raise ProtocolError("incorrect masking") + + length = head2 & 0b01111111 + if length == 126: + data = await reader(2) + (length,) = struct.unpack("!H", data) + elif length == 127: + data = await reader(8) + (length,) = struct.unpack("!Q", data) + if max_size is not None and length > max_size: + raise PayloadTooBig(f"over size limit ({length} > {max_size} bytes)") + if mask: + mask_bits = await reader(4) + + # Read the data. + data = await reader(length) + if mask: + data = apply_mask(data, mask_bits) + + frame = cls(fin, opcode, data, rsv1, rsv2, rsv3) + + if extensions is None: + extensions = [] + for extension in reversed(extensions): + frame = cls(*extension.decode(frame, max_size=max_size)) + + frame.check() + + return frame + + def write( + self, + write: Callable[[bytes], Any], + *, + mask: bool, + extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + ) -> None: + """ + Write a WebSocket frame. + + :param frame: frame to write + :param write: function that writes bytes + :param mask: whether the frame should be masked i.e. whether the write + happens on the client side + :param extensions: list of classes with an ``encode()`` method that + transform the frame and return a new frame; extensions are applied + in order + :raises ~websockets.exceptions.ProtocolError: if the frame + contains incorrect values + + """ + # The frame is written in a single call to write in order to prevent + # TCP fragmentation. See #68 for details. This also makes it safe to + # send frames concurrently from multiple coroutines. + write(self.serialize(mask=mask, extensions=extensions)) + + +# Backwards compatibility with previously documented public APIs +from ..frames import parse_close # isort:skip # noqa +from ..frames import prepare_ctrl as encode_data # isort:skip # noqa +from ..frames import prepare_data # isort:skip # noqa +from ..frames import serialize_close # isort:skip # noqa + + +# at the bottom to allow circular import, because Extension depends on Frame +import websockets.extensions.base # isort:skip # noqa diff --git a/src/websockets/handshake_legacy.py b/src/websockets/legacy/handshake.py similarity index 93% rename from src/websockets/handshake_legacy.py rename to src/websockets/legacy/handshake.py index d34ca5f7f..44da72d21 100644 --- a/src/websockets/handshake_legacy.py +++ b/src/websockets/legacy/handshake.py @@ -1,5 +1,5 @@ """ -:mod:`websockets.handshake` provides helpers for the WebSocket handshake. +:mod:`websockets.legacy.handshake` provides helpers for the WebSocket handshake. See `section 4 of RFC 6455`_. @@ -29,11 +29,11 @@ import binascii from typing import List -from .datastructures import Headers, MultipleValuesError -from .exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade -from .headers import parse_connection, parse_upgrade -from .typing import ConnectionOption, UpgradeProtocol -from .utils import accept_key as accept, generate_key +from ..datastructures import Headers, MultipleValuesError +from ..exceptions import InvalidHeader, InvalidHeaderValue, InvalidUpgrade +from ..headers import parse_connection, parse_upgrade +from ..typing import ConnectionOption, UpgradeProtocol +from ..utils import accept_key as accept, generate_key __all__ = ["build_request", "check_request", "build_response", "check_response"] diff --git a/src/websockets/http_legacy.py b/src/websockets/legacy/http.py similarity index 98% rename from src/websockets/http_legacy.py rename to src/websockets/legacy/http.py index 5afe5f898..c18e08e8d 100644 --- a/src/websockets/http_legacy.py +++ b/src/websockets/legacy/http.py @@ -2,8 +2,8 @@ import re from typing import Tuple -from .datastructures import Headers -from .exceptions import SecurityError +from ..datastructures import Headers +from ..exceptions import SecurityError __all__ = ["read_request", "read_response"] diff --git a/src/websockets/legacy/protocol.py b/src/websockets/legacy/protocol.py new file mode 100644 index 000000000..e4592b8a0 --- /dev/null +++ b/src/websockets/legacy/protocol.py @@ -0,0 +1,1459 @@ +""" +:mod:`websockets.legacy.protocol` handles WebSocket control and data frames. + +See `sections 4 to 8 of RFC 6455`_. + +.. _sections 4 to 8 of RFC 6455: http://tools.ietf.org/html/rfc6455#section-4 + +""" + +import asyncio +import codecs +import collections +import enum +import logging +import random +import struct +import sys +import warnings +from typing import ( + Any, + AsyncIterable, + AsyncIterator, + Awaitable, + Deque, + Dict, + Iterable, + List, + Mapping, + Optional, + Union, + cast, +) + +from ..datastructures import Headers +from ..exceptions import ( + ConnectionClosed, + ConnectionClosedError, + ConnectionClosedOK, + InvalidState, + PayloadTooBig, + ProtocolError, +) +from ..extensions.base import Extension +from ..frames import ( + OP_BINARY, + OP_CLOSE, + OP_CONT, + OP_PING, + OP_PONG, + OP_TEXT, + Opcode, + parse_close, + prepare_ctrl, + prepare_data, + serialize_close, +) +from ..typing import Data, Subprotocol +from .framing import Frame + + +__all__ = ["WebSocketCommonProtocol"] + +logger = logging.getLogger(__name__) + + +# A WebSocket connection goes through the following four states, in order: + + +class State(enum.IntEnum): + CONNECTING, OPEN, CLOSING, CLOSED = range(4) + + +# In order to ensure consistency, the code always checks the current value of +# WebSocketCommonProtocol.state before assigning a new value and never yields +# between the check and the assignment. + + +class WebSocketCommonProtocol(asyncio.Protocol): + """ + :class:`~asyncio.Protocol` subclass implementing the data transfer phase. + + Once the WebSocket connection is established, during the data transfer + phase, the protocol is almost symmetrical between the server side and the + client side. :class:`WebSocketCommonProtocol` implements logic that's + shared between servers and clients.. + + Subclasses such as + :class:`~websockets.legacy.server.WebSocketServerProtocol` and + :class:`~websockets.legacy.client.WebSocketClientProtocol` implement the + opening handshake, which is different between servers and clients. + + :class:`WebSocketCommonProtocol` performs four functions: + + * It runs a task that stores incoming data frames in a queue and makes + them available with the :meth:`recv` coroutine. + * It sends outgoing data frames with the :meth:`send` coroutine. + * It deals with control frames automatically. + * It performs the closing handshake. + + :class:`WebSocketCommonProtocol` supports asynchronous iteration:: + + async for message in websocket: + await process(message) + + The iterator yields incoming messages. It exits normally when the + connection is closed with the close code 1000 (OK) or 1001 (going away). + It raises a :exc:`~websockets.exceptions.ConnectionClosedError` exception + when the connection is closed with any other code. + + Once the connection is open, a `Ping frame`_ is sent every + ``ping_interval`` seconds. This serves as a keepalive. It helps keeping + the connection open, especially in the presence of proxies with short + timeouts on inactive connections. Set ``ping_interval`` to ``None`` to + disable this behavior. + + .. _Ping frame: https://tools.ietf.org/html/rfc6455#section-5.5.2 + + If the corresponding `Pong frame`_ isn't received within ``ping_timeout`` + seconds, the connection is considered unusable and is closed with + code 1011. This ensures that the remote endpoint remains responsive. Set + ``ping_timeout`` to ``None`` to disable this behavior. + + .. _Pong frame: https://tools.ietf.org/html/rfc6455#section-5.5.3 + + The ``close_timeout`` parameter defines a maximum wait time in seconds for + completing the closing handshake and terminating the TCP connection. + :meth:`close` completes in at most ``4 * close_timeout`` on the server + side and ``5 * close_timeout`` on the client side. + + ``close_timeout`` needs to be a parameter of the protocol because + ``websockets`` usually calls :meth:`close` implicitly: + + - on the server side, when the connection handler terminates, + - on the client side, when exiting the context manager for the connection. + + To apply a timeout to any other API, wrap it in :func:`~asyncio.wait_for`. + + The ``max_size`` parameter enforces the maximum size for incoming messages + in bytes. The default value is 1 MiB. ``None`` disables the limit. If a + message larger than the maximum size is received, :meth:`recv` will + raise :exc:`~websockets.exceptions.ConnectionClosedError` and the + connection will be closed with code 1009. + + The ``max_queue`` parameter sets the maximum length of the queue that + holds incoming messages. The default value is ``32``. ``None`` disables + the limit. Messages are added to an in-memory queue when they're received; + then :meth:`recv` pops from that queue. In order to prevent excessive + memory consumption when messages are received faster than they can be + processed, the queue must be bounded. If the queue fills up, the protocol + stops processing incoming data until :meth:`recv` is called. In this + situation, various receive buffers (at least in ``asyncio`` and in the OS) + will fill up, then the TCP receive window will shrink, slowing down + transmission to avoid packet loss. + + Since Python can use up to 4 bytes of memory to represent a single + character, each connection may use up to ``4 * max_size * max_queue`` + bytes of memory to store incoming messages. By default, this is 128 MiB. + You may want to lower the limits, depending on your application's + requirements. + + The ``read_limit`` argument sets the high-water limit of the buffer for + incoming bytes. The low-water limit is half the high-water limit. The + default value is 64 KiB, half of asyncio's default (based on the current + implementation of :class:`~asyncio.StreamReader`). + + The ``write_limit`` argument sets the high-water limit of the buffer for + outgoing bytes. The low-water limit is a quarter of the high-water limit. + The default value is 64 KiB, equal to asyncio's default (based on the + current implementation of ``FlowControlMixin``). + + As soon as the HTTP request and response in the opening handshake are + processed: + + * the request path is available in the :attr:`path` attribute; + * the request and response HTTP headers are available in the + :attr:`request_headers` and :attr:`response_headers` attributes, + which are :class:`~websockets.http.Headers` instances. + + If a subprotocol was negotiated, it's available in the :attr:`subprotocol` + attribute. + + Once the connection is closed, the code is available in the + :attr:`close_code` attribute and the reason in :attr:`close_reason`. + + All these attributes must be treated as read-only. + + """ + + # There are only two differences between the client-side and server-side + # behavior: masking the payload and closing the underlying TCP connection. + # Set is_client = True/False and side = "client"/"server" to pick a side. + is_client: bool + side: str = "undefined" + + def __init__( + self, + *, + ping_interval: Optional[float] = 20, + ping_timeout: Optional[float] = 20, + close_timeout: Optional[float] = None, + max_size: Optional[int] = 2 ** 20, + max_queue: Optional[int] = 2 ** 5, + read_limit: int = 2 ** 16, + write_limit: int = 2 ** 16, + loop: Optional[asyncio.AbstractEventLoop] = None, + # The following arguments are kept only for backwards compatibility. + host: Optional[str] = None, + port: Optional[int] = None, + secure: Optional[bool] = None, + legacy_recv: bool = False, + timeout: Optional[float] = None, + ) -> None: + # Backwards compatibility: close_timeout used to be called timeout. + if timeout is None: + timeout = 10 + else: + warnings.warn("rename timeout to close_timeout", DeprecationWarning) + # If both are specified, timeout is ignored. + if close_timeout is None: + close_timeout = timeout + + self.ping_interval = ping_interval + self.ping_timeout = ping_timeout + self.close_timeout = close_timeout + self.max_size = max_size + self.max_queue = max_queue + self.read_limit = read_limit + self.write_limit = write_limit + + if loop is None: + loop = asyncio.get_event_loop() + self.loop = loop + + self._host = host + self._port = port + self._secure = secure + self.legacy_recv = legacy_recv + + # Configure read buffer limits. The high-water limit is defined by + # ``self.read_limit``. The ``limit`` argument controls the line length + # limit and half the buffer limit of :class:`~asyncio.StreamReader`. + # That's why it must be set to half of ``self.read_limit``. + self.reader = asyncio.StreamReader(limit=read_limit // 2, loop=loop) + + # Copied from asyncio.FlowControlMixin + self._paused = False + self._drain_waiter: Optional[asyncio.Future[None]] = None + + self._drain_lock = asyncio.Lock( + loop=loop if sys.version_info[:2] < (3, 8) else None + ) + + # This class implements the data transfer and closing handshake, which + # are shared between the client-side and the server-side. + # Subclasses implement the opening handshake and, on success, execute + # :meth:`connection_open` to change the state to OPEN. + self.state = State.CONNECTING + logger.debug("%s - state = CONNECTING", self.side) + + # HTTP protocol parameters. + self.path: str + self.request_headers: Headers + self.response_headers: Headers + + # WebSocket protocol parameters. + self.extensions: List[Extension] = [] + self.subprotocol: Optional[Subprotocol] = None + + # The close code and reason are set when receiving a close frame or + # losing the TCP connection. + self.close_code: int + self.close_reason: str + + # Completed when the connection state becomes CLOSED. Translates the + # :meth:`connection_lost` callback to a :class:`~asyncio.Future` + # that can be awaited. (Other :class:`~asyncio.Protocol` callbacks are + # translated by ``self.stream_reader``). + self.connection_lost_waiter: asyncio.Future[None] = loop.create_future() + + # Queue of received messages. + self.messages: Deque[Data] = collections.deque() + self._pop_message_waiter: Optional[asyncio.Future[None]] = None + self._put_message_waiter: Optional[asyncio.Future[None]] = None + + # Protect sending fragmented messages. + self._fragmented_message_waiter: Optional[asyncio.Future[None]] = None + + # Mapping of ping IDs to pong waiters, in chronological order. + self.pings: Dict[bytes, asyncio.Future[None]] = {} + + # Task running the data transfer. + self.transfer_data_task: asyncio.Task[None] + + # Exception that occurred during data transfer, if any. + self.transfer_data_exc: Optional[BaseException] = None + + # Task sending keepalive pings. + self.keepalive_ping_task: asyncio.Task[None] + + # Task closing the TCP connection. + self.close_connection_task: asyncio.Task[None] + + # Copied from asyncio.FlowControlMixin + async def _drain_helper(self) -> None: # pragma: no cover + if self.connection_lost_waiter.done(): + raise ConnectionResetError("Connection lost") + if not self._paused: + return + waiter = self._drain_waiter + assert waiter is None or waiter.cancelled() + waiter = self.loop.create_future() + self._drain_waiter = waiter + await waiter + + # Copied from asyncio.StreamWriter + async def _drain(self) -> None: # pragma: no cover + if self.reader is not None: + exc = self.reader.exception() + if exc is not None: + raise exc + if self.transport is not None: + if self.transport.is_closing(): + # Yield to the event loop so connection_lost() may be + # called. Without this, _drain_helper() would return + # immediately, and code that calls + # write(...); yield from drain() + # in a loop would never call connection_lost(), so it + # would not see an error when the socket is closed. + await asyncio.sleep( + 0, loop=self.loop if sys.version_info[:2] < (3, 8) else None + ) + await self._drain_helper() + + def connection_open(self) -> None: + """ + Callback when the WebSocket opening handshake completes. + + Enter the OPEN state and start the data transfer phase. + + """ + # 4.1. The WebSocket Connection is Established. + assert self.state is State.CONNECTING + self.state = State.OPEN + logger.debug("%s - state = OPEN", self.side) + # Start the task that receives incoming WebSocket messages. + self.transfer_data_task = self.loop.create_task(self.transfer_data()) + # Start the task that sends pings at regular intervals. + self.keepalive_ping_task = self.loop.create_task(self.keepalive_ping()) + # Start the task that eventually closes the TCP connection. + self.close_connection_task = self.loop.create_task(self.close_connection()) + + @property + def host(self) -> Optional[str]: + alternative = "remote_address" if self.is_client else "local_address" + warnings.warn(f"use {alternative}[0] instead of host", DeprecationWarning) + return self._host + + @property + def port(self) -> Optional[int]: + alternative = "remote_address" if self.is_client else "local_address" + warnings.warn(f"use {alternative}[1] instead of port", DeprecationWarning) + return self._port + + @property + def secure(self) -> Optional[bool]: + warnings.warn("don't use secure", DeprecationWarning) + return self._secure + + # Public API + + @property + def local_address(self) -> Any: + """ + Local address of the connection as a ``(host, port)`` tuple. + + When the connection isn't open, ``local_address`` is ``None``. + + """ + try: + transport = self.transport + except AttributeError: + return None + else: + return transport.get_extra_info("sockname") + + @property + def remote_address(self) -> Any: + """ + Remote address of the connection as a ``(host, port)`` tuple. + + When the connection isn't open, ``remote_address`` is ``None``. + + """ + try: + transport = self.transport + except AttributeError: + return None + else: + return transport.get_extra_info("peername") + + @property + def open(self) -> bool: + """ + ``True`` when the connection is usable. + + It may be used to detect disconnections. However, this approach is + discouraged per the EAFP_ principle. + + When ``open`` is ``False``, using the connection raises a + :exc:`~websockets.exceptions.ConnectionClosed` exception. + + .. _EAFP: https://docs.python.org/3/glossary.html#term-eafp + + """ + return self.state is State.OPEN and not self.transfer_data_task.done() + + @property + def closed(self) -> bool: + """ + ``True`` once the connection is closed. + + Be aware that both :attr:`open` and :attr:`closed` are ``False`` during + the opening and closing sequences. + + """ + return self.state is State.CLOSED + + async def wait_closed(self) -> None: + """ + Wait until the connection is closed. + + This is identical to :attr:`closed`, except it can be awaited. + + This can make it easier to handle connection termination, regardless + of its cause, in tasks that interact with the WebSocket connection. + + """ + await asyncio.shield(self.connection_lost_waiter) + + async def __aiter__(self) -> AsyncIterator[Data]: + """ + Iterate on received messages. + + Exit normally when the connection is closed with code 1000 or 1001. + + Raise an exception in other cases. + + """ + try: + while True: + yield await self.recv() + except ConnectionClosedOK: + return + + async def recv(self) -> Data: + """ + Receive the next message. + + Return a :class:`str` for a text frame and :class:`bytes` for a binary + frame. + + When the end of the message stream is reached, :meth:`recv` raises + :exc:`~websockets.exceptions.ConnectionClosed`. Specifically, it + raises :exc:`~websockets.exceptions.ConnectionClosedOK` after a normal + connection closure and + :exc:`~websockets.exceptions.ConnectionClosedError` after a protocol + error or a network failure. + + .. versionchanged:: 3.0 + + :meth:`recv` used to return ``None`` instead. Refer to the + changelog for details. + + Canceling :meth:`recv` is safe. There's no risk of losing the next + message. The next invocation of :meth:`recv` will return it. This + makes it possible to enforce a timeout by wrapping :meth:`recv` in + :func:`~asyncio.wait_for`. + + :raises ~websockets.exceptions.ConnectionClosed: when the + connection is closed + :raises RuntimeError: if two coroutines call :meth:`recv` concurrently + + """ + if self._pop_message_waiter is not None: + raise RuntimeError( + "cannot call recv while another coroutine " + "is already waiting for the next message" + ) + + # Don't await self.ensure_open() here: + # - messages could be available in the queue even if the connection + # is closed; + # - messages could be received before the closing frame even if the + # connection is closing. + + # Wait until there's a message in the queue (if necessary) or the + # connection is closed. + while len(self.messages) <= 0: + pop_message_waiter: asyncio.Future[None] = self.loop.create_future() + self._pop_message_waiter = pop_message_waiter + try: + # If asyncio.wait() is canceled, it doesn't cancel + # pop_message_waiter and self.transfer_data_task. + await asyncio.wait( + [pop_message_waiter, self.transfer_data_task], + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + self._pop_message_waiter = None + + # If asyncio.wait(...) exited because self.transfer_data_task + # completed before receiving a new message, raise a suitable + # exception (or return None if legacy_recv is enabled). + if not pop_message_waiter.done(): + if self.legacy_recv: + return None # type: ignore + else: + # Wait until the connection is closed to raise + # ConnectionClosed with the correct code and reason. + await self.ensure_open() + + # Pop a message from the queue. + message = self.messages.popleft() + + # Notify transfer_data(). + if self._put_message_waiter is not None: + self._put_message_waiter.set_result(None) + self._put_message_waiter = None + + return message + + async def send( + self, message: Union[Data, Iterable[Data], AsyncIterable[Data]] + ) -> None: + """ + Send a message. + + A string (:class:`str`) is sent as a `Text frame`_. A bytestring or + bytes-like object (:class:`bytes`, :class:`bytearray`, or + :class:`memoryview`) is sent as a `Binary frame`_. + + .. _Text frame: https://tools.ietf.org/html/rfc6455#section-5.6 + .. _Binary frame: https://tools.ietf.org/html/rfc6455#section-5.6 + + :meth:`send` also accepts an iterable or an asynchronous iterable of + strings, bytestrings, or bytes-like objects. In that case the message + is fragmented. Each item is treated as a message fragment and sent in + its own frame. All items must be of the same type, or else + :meth:`send` will raise a :exc:`TypeError` and the connection will be + closed. + + :meth:`send` rejects dict-like objects because this is often an error. + If you wish to send the keys of a dict-like object as fragments, call + its :meth:`~dict.keys` method and pass the result to :meth:`send`. + + Canceling :meth:`send` is discouraged. Instead, you should close the + connection with :meth:`close`. Indeed, there only two situations where + :meth:`send` yields control to the event loop: + + 1. The write buffer is full. If you don't want to wait until enough + data is sent, your only alternative is to close the connection. + :meth:`close` will likely time out then abort the TCP connection. + 2. ``message`` is an asynchronous iterator. Stopping in the middle of + a fragmented message will cause a protocol error. Closing the + connection has the same effect. + + :raises TypeError: for unsupported inputs + + """ + await self.ensure_open() + + # While sending a fragmented message, prevent sending other messages + # until all fragments are sent. + while self._fragmented_message_waiter is not None: + await asyncio.shield(self._fragmented_message_waiter) + + # Unfragmented message -- this case must be handled first because + # strings and bytes-like objects are iterable. + + if isinstance(message, (str, bytes, bytearray, memoryview)): + opcode, data = prepare_data(message) + await self.write_frame(True, opcode, data) + + # Catch a common mistake -- passing a dict to send(). + + elif isinstance(message, Mapping): + raise TypeError("data is a dict-like object") + + # Fragmented message -- regular iterator. + + elif isinstance(message, Iterable): + + # Work around https://github.com/python/mypy/issues/6227 + message = cast(Iterable[Data], message) + + iter_message = iter(message) + try: + message_chunk = next(iter_message) + except StopIteration: + return + opcode, data = prepare_data(message_chunk) + + self._fragmented_message_waiter = asyncio.Future() + try: + # First fragment. + await self.write_frame(False, opcode, data) + + # Other fragments. + for message_chunk in iter_message: + confirm_opcode, data = prepare_data(message_chunk) + if confirm_opcode != opcode: + raise TypeError("data contains inconsistent types") + await self.write_frame(False, OP_CONT, data) + + # Final fragment. + await self.write_frame(True, OP_CONT, b"") + + except Exception: + # We're half-way through a fragmented message and we can't + # complete it. This makes the connection unusable. + self.fail_connection(1011) + raise + + finally: + self._fragmented_message_waiter.set_result(None) + self._fragmented_message_waiter = None + + # Fragmented message -- asynchronous iterator + + elif isinstance(message, AsyncIterable): + # aiter_message = aiter(message) without aiter + # https://github.com/python/mypy/issues/5738 + aiter_message = type(message).__aiter__(message) # type: ignore + try: + # message_chunk = anext(aiter_message) without anext + # https://github.com/python/mypy/issues/5738 + message_chunk = await type(aiter_message).__anext__( # type: ignore + aiter_message + ) + except StopAsyncIteration: + return + opcode, data = prepare_data(message_chunk) + + self._fragmented_message_waiter = asyncio.Future() + try: + # First fragment. + await self.write_frame(False, opcode, data) + + # Other fragments. + # https://github.com/python/mypy/issues/5738 + async for message_chunk in aiter_message: # type: ignore + confirm_opcode, data = prepare_data(message_chunk) + if confirm_opcode != opcode: + raise TypeError("data contains inconsistent types") + await self.write_frame(False, OP_CONT, data) + + # Final fragment. + await self.write_frame(True, OP_CONT, b"") + + except Exception: + # We're half-way through a fragmented message and we can't + # complete it. This makes the connection unusable. + self.fail_connection(1011) + raise + + finally: + self._fragmented_message_waiter.set_result(None) + self._fragmented_message_waiter = None + + else: + raise TypeError("data must be bytes, str, or iterable") + + async def close(self, code: int = 1000, reason: str = "") -> None: + """ + Perform the closing handshake. + + :meth:`close` waits for the other end to complete the handshake and + for the TCP connection to terminate. As a consequence, there's no need + to await :meth:`wait_closed`; :meth:`close` already does it. + + :meth:`close` is idempotent: it doesn't do anything once the + connection is closed. + + Wrapping :func:`close` in :func:`~asyncio.create_task` is safe, given + that errors during connection termination aren't particularly useful. + + Canceling :meth:`close` is discouraged. If it takes too long, you can + set a shorter ``close_timeout``. If you don't want to wait, let the + Python process exit, then the OS will close the TCP connection. + + :param code: WebSocket close code + :param reason: WebSocket close reason + + """ + try: + await asyncio.wait_for( + self.write_close_frame(serialize_close(code, reason)), + self.close_timeout, + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + ) + except asyncio.TimeoutError: + # If the close frame cannot be sent because the send buffers + # are full, the closing handshake won't complete anyway. + # Fail the connection to shut down faster. + self.fail_connection() + + # If no close frame is received within the timeout, wait_for() cancels + # the data transfer task and raises TimeoutError. + + # If close() is called multiple times concurrently and one of these + # calls hits the timeout, the data transfer task will be cancelled. + # Other calls will receive a CancelledError here. + + try: + # If close() is canceled during the wait, self.transfer_data_task + # is canceled before the timeout elapses. + await asyncio.wait_for( + self.transfer_data_task, + self.close_timeout, + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + ) + except (asyncio.TimeoutError, asyncio.CancelledError): + pass + + # Wait for the close connection task to close the TCP connection. + await asyncio.shield(self.close_connection_task) + + async def ping(self, data: Optional[Data] = None) -> Awaitable[None]: + """ + Send a ping. + + Return a :class:`~asyncio.Future` that will be completed when the + corresponding pong is received. You can ignore it if you don't intend + to wait. + + A ping may serve as a keepalive or as a check that the remote endpoint + received all messages up to this point:: + + pong_waiter = await ws.ping() + await pong_waiter # only if you want to wait for the pong + + By default, the ping contains four random bytes. This payload may be + overridden with the optional ``data`` argument which must be a string + (which will be encoded to UTF-8) or a bytes-like object. + + Canceling :meth:`ping` is discouraged. If :meth:`ping` doesn't return + immediately, it means the write buffer is full. If you don't want to + wait, you should close the connection. + + Canceling the :class:`~asyncio.Future` returned by :meth:`ping` has no + effect. + + """ + await self.ensure_open() + + if data is not None: + data = prepare_ctrl(data) + + # Protect against duplicates if a payload is explicitly set. + if data in self.pings: + raise ValueError("already waiting for a pong with the same data") + + # Generate a unique random payload otherwise. + while data is None or data in self.pings: + data = struct.pack("!I", random.getrandbits(32)) + + self.pings[data] = self.loop.create_future() + + await self.write_frame(True, OP_PING, data) + + return asyncio.shield(self.pings[data]) + + async def pong(self, data: Data = b"") -> None: + """ + Send a pong. + + An unsolicited pong may serve as a unidirectional heartbeat. + + The payload may be set with the optional ``data`` argument which must + be a string (which will be encoded to UTF-8) or a bytes-like object. + + Canceling :meth:`pong` is discouraged for the same reason as + :meth:`ping`. + + """ + await self.ensure_open() + + data = prepare_ctrl(data) + + await self.write_frame(True, OP_PONG, data) + + # Private methods - no guarantees. + + def connection_closed_exc(self) -> ConnectionClosed: + exception: ConnectionClosed + if self.close_code == 1000 or self.close_code == 1001: + exception = ConnectionClosedOK(self.close_code, self.close_reason) + else: + exception = ConnectionClosedError(self.close_code, self.close_reason) + # Chain to the exception that terminated data transfer, if any. + exception.__cause__ = self.transfer_data_exc + return exception + + async def ensure_open(self) -> None: + """ + Check that the WebSocket connection is open. + + Raise :exc:`~websockets.exceptions.ConnectionClosed` if it isn't. + + """ + # Handle cases from most common to least common for performance. + if self.state is State.OPEN: + # If self.transfer_data_task exited without a closing handshake, + # self.close_connection_task may be closing the connection, going + # straight from OPEN to CLOSED. + if self.transfer_data_task.done(): + await asyncio.shield(self.close_connection_task) + raise self.connection_closed_exc() + else: + return + + if self.state is State.CLOSED: + raise self.connection_closed_exc() + + if self.state is State.CLOSING: + # If we started the closing handshake, wait for its completion to + # get the proper close code and reason. self.close_connection_task + # will complete within 4 or 5 * close_timeout after close(). The + # CLOSING state also occurs when failing the connection. In that + # case self.close_connection_task will complete even faster. + await asyncio.shield(self.close_connection_task) + raise self.connection_closed_exc() + + # Control may only reach this point in buggy third-party subclasses. + assert self.state is State.CONNECTING + raise InvalidState("WebSocket connection isn't established yet") + + async def transfer_data(self) -> None: + """ + Read incoming messages and put them in a queue. + + This coroutine runs in a task until the closing handshake is started. + + """ + try: + while True: + message = await self.read_message() + + # Exit the loop when receiving a close frame. + if message is None: + break + + # Wait until there's room in the queue (if necessary). + if self.max_queue is not None: + while len(self.messages) >= self.max_queue: + self._put_message_waiter = self.loop.create_future() + try: + await asyncio.shield(self._put_message_waiter) + finally: + self._put_message_waiter = None + + # Put the message in the queue. + self.messages.append(message) + + # Notify recv(). + if self._pop_message_waiter is not None: + self._pop_message_waiter.set_result(None) + self._pop_message_waiter = None + + except asyncio.CancelledError as exc: + self.transfer_data_exc = exc + # If fail_connection() cancels this task, avoid logging the error + # twice and failing the connection again. + raise + + except ProtocolError as exc: + self.transfer_data_exc = exc + self.fail_connection(1002) + + except (ConnectionError, TimeoutError, EOFError) as exc: + # Reading data with self.reader.readexactly may raise: + # - most subclasses of ConnectionError if the TCP connection + # breaks, is reset, or is aborted; + # - TimeoutError if the TCP connection times out; + # - IncompleteReadError, a subclass of EOFError, if fewer + # bytes are available than requested. + self.transfer_data_exc = exc + self.fail_connection(1006) + + except UnicodeDecodeError as exc: + self.transfer_data_exc = exc + self.fail_connection(1007) + + except PayloadTooBig as exc: + self.transfer_data_exc = exc + self.fail_connection(1009) + + except Exception as exc: + # This shouldn't happen often because exceptions expected under + # regular circumstances are handled above. If it does, consider + # catching and handling more exceptions. + logger.error("Error in data transfer", exc_info=True) + + self.transfer_data_exc = exc + self.fail_connection(1011) + + async def read_message(self) -> Optional[Data]: + """ + Read a single message from the connection. + + Re-assemble data frames if the message is fragmented. + + Return ``None`` when the closing handshake is started. + + """ + frame = await self.read_data_frame(max_size=self.max_size) + + # A close frame was received. + if frame is None: + return None + + if frame.opcode == OP_TEXT: + text = True + elif frame.opcode == OP_BINARY: + text = False + else: # frame.opcode == OP_CONT + raise ProtocolError("unexpected opcode") + + # Shortcut for the common case - no fragmentation + if frame.fin: + return frame.data.decode("utf-8") if text else frame.data + + # 5.4. Fragmentation + chunks: List[Data] = [] + max_size = self.max_size + if text: + decoder_factory = codecs.getincrementaldecoder("utf-8") + decoder = decoder_factory(errors="strict") + if max_size is None: + + def append(frame: Frame) -> None: + nonlocal chunks + chunks.append(decoder.decode(frame.data, frame.fin)) + + else: + + def append(frame: Frame) -> None: + nonlocal chunks, max_size + chunks.append(decoder.decode(frame.data, frame.fin)) + assert isinstance(max_size, int) + max_size -= len(frame.data) + + else: + if max_size is None: + + def append(frame: Frame) -> None: + nonlocal chunks + chunks.append(frame.data) + + else: + + def append(frame: Frame) -> None: + nonlocal chunks, max_size + chunks.append(frame.data) + assert isinstance(max_size, int) + max_size -= len(frame.data) + + append(frame) + + while not frame.fin: + frame = await self.read_data_frame(max_size=max_size) + if frame is None: + raise ProtocolError("incomplete fragmented message") + if frame.opcode != OP_CONT: + raise ProtocolError("unexpected opcode") + append(frame) + + # mypy cannot figure out that chunks have the proper type. + return ("" if text else b"").join(chunks) # type: ignore + + async def read_data_frame(self, max_size: Optional[int]) -> Optional[Frame]: + """ + Read a single data frame from the connection. + + Process control frames received before the next data frame. + + Return ``None`` if a close frame is encountered before any data frame. + + """ + # 6.2. Receiving Data + while True: + frame = await self.read_frame(max_size) + + # 5.5. Control Frames + if frame.opcode == OP_CLOSE: + # 7.1.5. The WebSocket Connection Close Code + # 7.1.6. The WebSocket Connection Close Reason + self.close_code, self.close_reason = parse_close(frame.data) + try: + # Echo the original data instead of re-serializing it with + # serialize_close() because that fails when the close frame + # is empty and parse_close() synthetizes a 1005 close code. + await self.write_close_frame(frame.data) + except ConnectionClosed: + # It doesn't really matter if the connection was closed + # before we could send back a close frame. + pass + return None + + elif frame.opcode == OP_PING: + # Answer pings. + ping_hex = frame.data.hex() or "[empty]" + logger.debug( + "%s - received ping, sending pong: %s", self.side, ping_hex + ) + await self.pong(frame.data) + + elif frame.opcode == OP_PONG: + # Acknowledge pings on solicited pongs. + if frame.data in self.pings: + logger.debug( + "%s - received solicited pong: %s", + self.side, + frame.data.hex() or "[empty]", + ) + # Acknowledge all pings up to the one matching this pong. + ping_id = None + ping_ids = [] + for ping_id, ping in self.pings.items(): + ping_ids.append(ping_id) + if not ping.done(): + ping.set_result(None) + if ping_id == frame.data: + break + else: # pragma: no cover + assert False, "ping_id is in self.pings" + # Remove acknowledged pings from self.pings. + for ping_id in ping_ids: + del self.pings[ping_id] + ping_ids = ping_ids[:-1] + if ping_ids: + pings_hex = ", ".join( + ping_id.hex() or "[empty]" for ping_id in ping_ids + ) + plural = "s" if len(ping_ids) > 1 else "" + logger.debug( + "%s - acknowledged previous ping%s: %s", + self.side, + plural, + pings_hex, + ) + else: + logger.debug( + "%s - received unsolicited pong: %s", + self.side, + frame.data.hex() or "[empty]", + ) + + # 5.6. Data Frames + else: + return frame + + async def read_frame(self, max_size: Optional[int]) -> Frame: + """ + Read a single frame from the connection. + + """ + frame = await Frame.read( + self.reader.readexactly, + mask=not self.is_client, + max_size=max_size, + extensions=self.extensions, + ) + logger.debug("%s < %r", self.side, frame) + return frame + + async def write_frame( + self, fin: bool, opcode: int, data: bytes, *, _expected_state: int = State.OPEN + ) -> None: + # Defensive assertion for protocol compliance. + if self.state is not _expected_state: # pragma: no cover + raise InvalidState( + f"Cannot write to a WebSocket in the {self.state.name} state" + ) + + frame = Frame(fin, Opcode(opcode), data) + logger.debug("%s > %r", self.side, frame) + frame.write( + self.transport.write, mask=self.is_client, extensions=self.extensions + ) + + try: + # drain() cannot be called concurrently by multiple coroutines: + # http://bugs.python.org/issue29930. Remove this lock when no + # version of Python where this bugs exists is supported anymore. + async with self._drain_lock: + # Handle flow control automatically. + await self._drain() + except ConnectionError: + # Terminate the connection if the socket died. + self.fail_connection() + # Wait until the connection is closed to raise ConnectionClosed + # with the correct code and reason. + await self.ensure_open() + + async def write_close_frame(self, data: bytes = b"") -> None: + """ + Write a close frame if and only if the connection state is OPEN. + + This dedicated coroutine must be used for writing close frames to + ensure that at most one close frame is sent on a given connection. + + """ + # Test and set the connection state before sending the close frame to + # avoid sending two frames in case of concurrent calls. + if self.state is State.OPEN: + # 7.1.3. The WebSocket Closing Handshake is Started + self.state = State.CLOSING + logger.debug("%s - state = CLOSING", self.side) + + # 7.1.2. Start the WebSocket Closing Handshake + await self.write_frame(True, OP_CLOSE, data, _expected_state=State.CLOSING) + + async def keepalive_ping(self) -> None: + """ + Send a Ping frame and wait for a Pong frame at regular intervals. + + This coroutine exits when the connection terminates and one of the + following happens: + + - :meth:`ping` raises :exc:`ConnectionClosed`, or + - :meth:`close_connection` cancels :attr:`keepalive_ping_task`. + + """ + if self.ping_interval is None: + return + + try: + while True: + await asyncio.sleep( + self.ping_interval, + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + ) + + # ping() raises CancelledError if the connection is closed, + # when close_connection() cancels self.keepalive_ping_task. + + # ping() raises ConnectionClosed if the connection is lost, + # when connection_lost() calls abort_pings(). + + pong_waiter = await self.ping() + + if self.ping_timeout is not None: + try: + await asyncio.wait_for( + pong_waiter, + self.ping_timeout, + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + ) + except asyncio.TimeoutError: + logger.debug("%s ! timed out waiting for pong", self.side) + self.fail_connection(1011) + break + + # Remove this branch when dropping support for Python < 3.8 + # because CancelledError no longer inherits Exception. + except asyncio.CancelledError: + raise + + except ConnectionClosed: + pass + + except Exception: + logger.warning("Unexpected exception in keepalive ping task", exc_info=True) + + async def close_connection(self) -> None: + """ + 7.1.1. Close the WebSocket Connection + + When the opening handshake succeeds, :meth:`connection_open` starts + this coroutine in a task. It waits for the data transfer phase to + complete then it closes the TCP connection cleanly. + + When the opening handshake fails, :meth:`fail_connection` does the + same. There's no data transfer phase in that case. + + """ + try: + # Wait for the data transfer phase to complete. + if hasattr(self, "transfer_data_task"): + try: + await self.transfer_data_task + except asyncio.CancelledError: + pass + + # Cancel the keepalive ping task. + if hasattr(self, "keepalive_ping_task"): + self.keepalive_ping_task.cancel() + + # A client should wait for a TCP close from the server. + if self.is_client and hasattr(self, "transfer_data_task"): + if await self.wait_for_connection_lost(): + # Coverage marks this line as a partially executed branch. + # I supect a bug in coverage. Ignore it for now. + return # pragma: no cover + logger.debug("%s ! timed out waiting for TCP close", self.side) + + # Half-close the TCP connection if possible (when there's no TLS). + if self.transport.can_write_eof(): + logger.debug("%s x half-closing TCP connection", self.side) + self.transport.write_eof() + + if await self.wait_for_connection_lost(): + # Coverage marks this line as a partially executed branch. + # I supect a bug in coverage. Ignore it for now. + return # pragma: no cover + logger.debug("%s ! timed out waiting for TCP close", self.side) + + finally: + # The try/finally ensures that the transport never remains open, + # even if this coroutine is canceled (for example). + + # If connection_lost() was called, the TCP connection is closed. + # However, if TLS is enabled, the transport still needs closing. + # Else asyncio complains: ResourceWarning: unclosed transport. + if self.connection_lost_waiter.done() and self.transport.is_closing(): + return + + # Close the TCP connection. Buffers are flushed asynchronously. + logger.debug("%s x closing TCP connection", self.side) + self.transport.close() + + if await self.wait_for_connection_lost(): + return + logger.debug("%s ! timed out waiting for TCP close", self.side) + + # Abort the TCP connection. Buffers are discarded. + logger.debug("%s x aborting TCP connection", self.side) + self.transport.abort() + + # connection_lost() is called quickly after aborting. + # Coverage marks this line as a partially executed branch. + # I supect a bug in coverage. Ignore it for now. + await self.wait_for_connection_lost() # pragma: no cover + + async def wait_for_connection_lost(self) -> bool: + """ + Wait until the TCP connection is closed or ``self.close_timeout`` elapses. + + Return ``True`` if the connection is closed and ``False`` otherwise. + + """ + if not self.connection_lost_waiter.done(): + try: + await asyncio.wait_for( + asyncio.shield(self.connection_lost_waiter), + self.close_timeout, + loop=self.loop if sys.version_info[:2] < (3, 8) else None, + ) + except asyncio.TimeoutError: + pass + # Re-check self.connection_lost_waiter.done() synchronously because + # connection_lost() could run between the moment the timeout occurs + # and the moment this coroutine resumes running. + return self.connection_lost_waiter.done() + + def fail_connection(self, code: int = 1006, reason: str = "") -> None: + """ + 7.1.7. Fail the WebSocket Connection + + This requires: + + 1. Stopping all processing of incoming data, which means cancelling + :attr:`transfer_data_task`. The close code will be 1006 unless a + close frame was received earlier. + + 2. Sending a close frame with an appropriate code if the opening + handshake succeeded and the other side is likely to process it. + + 3. Closing the connection. :meth:`close_connection` takes care of + this once :attr:`transfer_data_task` exits after being canceled. + + (The specification describes these steps in the opposite order.) + + """ + logger.debug( + "%s ! failing %s WebSocket connection with code %d", + self.side, + self.state.name, + code, + ) + + # Cancel transfer_data_task if the opening handshake succeeded. + # cancel() is idempotent and ignored if the task is done already. + if hasattr(self, "transfer_data_task"): + self.transfer_data_task.cancel() + + # Send a close frame when the state is OPEN (a close frame was already + # sent if it's CLOSING), except when failing the connection because of + # an error reading from or writing to the network. + # Don't send a close frame if the connection is broken. + if code != 1006 and self.state is State.OPEN: + + frame_data = serialize_close(code, reason) + + # Write the close frame without draining the write buffer. + + # Keeping fail_connection() synchronous guarantees it can't + # get stuck and simplifies the implementation of the callers. + # Not drainig the write buffer is acceptable in this context. + + # This duplicates a few lines of code from write_close_frame() + # and write_frame(). + + self.state = State.CLOSING + logger.debug("%s - state = CLOSING", self.side) + + frame = Frame(True, OP_CLOSE, frame_data) + logger.debug("%s > %r", self.side, frame) + frame.write( + self.transport.write, mask=self.is_client, extensions=self.extensions + ) + + # Start close_connection_task if the opening handshake didn't succeed. + if not hasattr(self, "close_connection_task"): + self.close_connection_task = self.loop.create_task(self.close_connection()) + + def abort_pings(self) -> None: + """ + Raise ConnectionClosed in pending keepalive pings. + + They'll never receive a pong once the connection is closed. + + """ + assert self.state is State.CLOSED + exc = self.connection_closed_exc() + + for ping in self.pings.values(): + ping.set_exception(exc) + # If the exception is never retrieved, it will be logged when ping + # is garbage-collected. This is confusing for users. + # Given that ping is done (with an exception), canceling it does + # nothing, but it prevents logging the exception. + ping.cancel() + + if self.pings: + pings_hex = ", ".join(ping_id.hex() or "[empty]" for ping_id in self.pings) + plural = "s" if len(self.pings) > 1 else "" + logger.debug( + "%s - aborted pending ping%s: %s", self.side, plural, pings_hex + ) + + # asyncio.Protocol methods + + def connection_made(self, transport: asyncio.BaseTransport) -> None: + """ + Configure write buffer limits. + + The high-water limit is defined by ``self.write_limit``. + + The low-water limit currently defaults to ``self.write_limit // 4`` in + :meth:`~asyncio.WriteTransport.set_write_buffer_limits`, which should + be all right for reasonable use cases of this library. + + This is the earliest point where we can get hold of the transport, + which means it's the best point for configuring it. + + """ + logger.debug("%s - event = connection_made(%s)", self.side, transport) + + transport = cast(asyncio.Transport, transport) + transport.set_write_buffer_limits(self.write_limit) + self.transport = transport + + # Copied from asyncio.StreamReaderProtocol + self.reader.set_transport(transport) + + def connection_lost(self, exc: Optional[Exception]) -> None: + """ + 7.1.4. The WebSocket Connection is Closed. + + """ + logger.debug("%s - event = connection_lost(%s)", self.side, exc) + self.state = State.CLOSED + logger.debug("%s - state = CLOSED", self.side) + if not hasattr(self, "close_code"): + self.close_code = 1006 + if not hasattr(self, "close_reason"): + self.close_reason = "" + logger.debug( + "%s x code = %d, reason = %s", + self.side, + self.close_code, + self.close_reason or "[no reason]", + ) + self.abort_pings() + # If self.connection_lost_waiter isn't pending, that's a bug, because: + # - it's set only here in connection_lost() which is called only once; + # - it must never be canceled. + self.connection_lost_waiter.set_result(None) + + if True: # pragma: no cover + + # Copied from asyncio.StreamReaderProtocol + if self.reader is not None: + if exc is None: + self.reader.feed_eof() + else: + self.reader.set_exception(exc) + + # Copied from asyncio.FlowControlMixin + # Wake up the writer if currently paused. + if not self._paused: + return + waiter = self._drain_waiter + if waiter is None: + return + self._drain_waiter = None + if waiter.done(): + return + if exc is None: + waiter.set_result(None) + else: + waiter.set_exception(exc) + + def pause_writing(self) -> None: # pragma: no cover + assert not self._paused + self._paused = True + + def resume_writing(self) -> None: # pragma: no cover + assert self._paused + self._paused = False + + waiter = self._drain_waiter + if waiter is not None: + self._drain_waiter = None + if not waiter.done(): + waiter.set_result(None) + + def data_received(self, data: bytes) -> None: + logger.debug("%s - event = data_received(<%d bytes>)", self.side, len(data)) + self.reader.feed_data(data) + + def eof_received(self) -> None: + """ + Close the transport after receiving EOF. + + The WebSocket protocol has its own closing handshake: endpoints close + the TCP or TLS connection after sending and receiving a close frame. + + As a consequence, they never need to write after receiving EOF, so + there's no reason to keep the transport open by returning ``True``. + + Besides, that doesn't work on TLS connections. + + """ + logger.debug("%s - event = eof_received()", self.side) + self.reader.feed_eof() diff --git a/src/websockets/asyncio_server.py b/src/websockets/legacy/server.py similarity index 97% rename from src/websockets/asyncio_server.py rename to src/websockets/legacy/server.py index 79ceddf4b..4dea9459d 100644 --- a/src/websockets/asyncio_server.py +++ b/src/websockets/legacy/server.py @@ -1,5 +1,5 @@ """ -:mod:`websockets.server` defines the WebSocket server APIs. +:mod:`websockets.legacy.server` defines the WebSocket server APIs. """ @@ -28,8 +28,8 @@ cast, ) -from .datastructures import Headers, HeadersLike, MultipleValuesError -from .exceptions import ( +from ..datastructures import Headers, HeadersLike, MultipleValuesError +from ..exceptions import ( AbortHandshake, InvalidHandshake, InvalidHeader, @@ -38,14 +38,14 @@ InvalidUpgrade, NegotiationError, ) -from .extensions.base import Extension, ServerExtensionFactory -from .extensions.permessage_deflate import enable_server_permessage_deflate -from .handshake_legacy import build_response, check_request -from .headers import build_extension, parse_extension, parse_subprotocol -from .http import USER_AGENT -from .http_legacy import read_request +from ..extensions.base import Extension, ServerExtensionFactory +from ..extensions.permessage_deflate import enable_server_permessage_deflate +from ..headers import build_extension, parse_extension, parse_subprotocol +from ..http import USER_AGENT +from ..typing import ExtensionHeader, Origin, Subprotocol +from .handshake import build_response, check_request +from .http import read_request from .protocol import WebSocketCommonProtocol -from .typing import ExtensionHeader, Origin, Subprotocol __all__ = ["serve", "unix_serve", "WebSocketServerProtocol", "WebSocketServer"] @@ -598,7 +598,7 @@ async def handshake( class WebSocketServer: """ - WebSocket server returned by :func:`~websockets.server.serve`. + WebSocket server returned by :func:`serve`. This class provides the same interface as :class:`~asyncio.AbstractServer`, namely the @@ -770,9 +770,9 @@ class Serve: performs the closing handshake and closes the connection. Awaiting :func:`serve` yields a :class:`WebSocketServer`. This instance - provides :meth:`~websockets.server.WebSocketServer.close` and - :meth:`~websockets.server.WebSocketServer.wait_closed` methods for - terminating the server and cleaning up its resources. + provides :meth:`~WebSocketServer.close` and + :meth:`~WebSocketServer.wait_closed` methods for terminating the server + and cleaning up its resources. When a server is closed with :meth:`~WebSocketServer.close`, it closes all connections with close code 1001 (going away). Connections handlers, which @@ -835,11 +835,11 @@ class Serve: :meth:`~WebSocketServerProtocol.select_subprotocol` for details Since there's no useful way to propagate exceptions triggered in handlers, - they're sent to the ``'websockets.asyncio_server'`` logger instead. + they're sent to the ``'websockets.legacy.server'`` logger instead. Debugging is much easier if you configure logging to print them:: import logging - logger = logging.getLogger("websockets.asyncio_server") + logger = logging.getLogger("websockets.legacy.server") logger.setLevel(logging.ERROR) logger.addHandler(logging.StreamHandler()) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py index 1552fb060..287f92a57 100644 --- a/src/websockets/protocol.py +++ b/src/websockets/protocol.py @@ -1,1465 +1 @@ -""" -:mod:`websockets.protocol` handles WebSocket control and data frames. - -See `sections 4 to 8 of RFC 6455`_. - -.. _sections 4 to 8 of RFC 6455: http://tools.ietf.org/html/rfc6455#section-4 - -""" - -import asyncio -import codecs -import collections -import enum -import logging -import random -import struct -import sys -import warnings -from typing import ( - Any, - AsyncIterable, - AsyncIterator, - Awaitable, - Deque, - Dict, - Iterable, - List, - Mapping, - Optional, - Union, - cast, -) - -from .datastructures import Headers -from .exceptions import ( - ConnectionClosed, - ConnectionClosedError, - ConnectionClosedOK, - InvalidState, - PayloadTooBig, - ProtocolError, -) -from .extensions.base import Extension -from .frames import ( - OP_BINARY, - OP_CLOSE, - OP_CONT, - OP_PING, - OP_PONG, - OP_TEXT, - Opcode, - parse_close, - prepare_ctrl, - prepare_data, - serialize_close, -) - - -with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "websockets.framing is deprecated", DeprecationWarning - ) - from .framing import Frame - -from .typing import Data, Subprotocol - - -__all__ = ["WebSocketCommonProtocol"] - -logger = logging.getLogger(__name__) - - -# A WebSocket connection goes through the following four states, in order: - - -class State(enum.IntEnum): - CONNECTING, OPEN, CLOSING, CLOSED = range(4) - - -# In order to ensure consistency, the code always checks the current value of -# WebSocketCommonProtocol.state before assigning a new value and never yields -# between the check and the assignment. - - -class WebSocketCommonProtocol(asyncio.Protocol): - """ - :class:`~asyncio.Protocol` subclass implementing the data transfer phase. - - Once the WebSocket connection is established, during the data transfer - phase, the protocol is almost symmetrical between the server side and the - client side. :class:`WebSocketCommonProtocol` implements logic that's - shared between servers and clients.. - - Subclasses such as :class:`~websockets.server.WebSocketServerProtocol` and - :class:`~websockets.client.WebSocketClientProtocol` implement the opening - handshake, which is different between servers and clients. - - :class:`WebSocketCommonProtocol` performs four functions: - - * It runs a task that stores incoming data frames in a queue and makes - them available with the :meth:`recv` coroutine. - * It sends outgoing data frames with the :meth:`send` coroutine. - * It deals with control frames automatically. - * It performs the closing handshake. - - :class:`WebSocketCommonProtocol` supports asynchronous iteration:: - - async for message in websocket: - await process(message) - - The iterator yields incoming messages. It exits normally when the - connection is closed with the close code 1000 (OK) or 1001 (going away). - It raises a :exc:`~websockets.exceptions.ConnectionClosedError` exception - when the connection is closed with any other code. - - Once the connection is open, a `Ping frame`_ is sent every - ``ping_interval`` seconds. This serves as a keepalive. It helps keeping - the connection open, especially in the presence of proxies with short - timeouts on inactive connections. Set ``ping_interval`` to ``None`` to - disable this behavior. - - .. _Ping frame: https://tools.ietf.org/html/rfc6455#section-5.5.2 - - If the corresponding `Pong frame`_ isn't received within ``ping_timeout`` - seconds, the connection is considered unusable and is closed with - code 1011. This ensures that the remote endpoint remains responsive. Set - ``ping_timeout`` to ``None`` to disable this behavior. - - .. _Pong frame: https://tools.ietf.org/html/rfc6455#section-5.5.3 - - The ``close_timeout`` parameter defines a maximum wait time in seconds for - completing the closing handshake and terminating the TCP connection. - :meth:`close` completes in at most ``4 * close_timeout`` on the server - side and ``5 * close_timeout`` on the client side. - - ``close_timeout`` needs to be a parameter of the protocol because - ``websockets`` usually calls :meth:`close` implicitly: - - - on the server side, when the connection handler terminates, - - on the client side, when exiting the context manager for the connection. - - To apply a timeout to any other API, wrap it in :func:`~asyncio.wait_for`. - - The ``max_size`` parameter enforces the maximum size for incoming messages - in bytes. The default value is 1 MiB. ``None`` disables the limit. If a - message larger than the maximum size is received, :meth:`recv` will - raise :exc:`~websockets.exceptions.ConnectionClosedError` and the - connection will be closed with code 1009. - - The ``max_queue`` parameter sets the maximum length of the queue that - holds incoming messages. The default value is ``32``. ``None`` disables - the limit. Messages are added to an in-memory queue when they're received; - then :meth:`recv` pops from that queue. In order to prevent excessive - memory consumption when messages are received faster than they can be - processed, the queue must be bounded. If the queue fills up, the protocol - stops processing incoming data until :meth:`recv` is called. In this - situation, various receive buffers (at least in ``asyncio`` and in the OS) - will fill up, then the TCP receive window will shrink, slowing down - transmission to avoid packet loss. - - Since Python can use up to 4 bytes of memory to represent a single - character, each connection may use up to ``4 * max_size * max_queue`` - bytes of memory to store incoming messages. By default, this is 128 MiB. - You may want to lower the limits, depending on your application's - requirements. - - The ``read_limit`` argument sets the high-water limit of the buffer for - incoming bytes. The low-water limit is half the high-water limit. The - default value is 64 KiB, half of asyncio's default (based on the current - implementation of :class:`~asyncio.StreamReader`). - - The ``write_limit`` argument sets the high-water limit of the buffer for - outgoing bytes. The low-water limit is a quarter of the high-water limit. - The default value is 64 KiB, equal to asyncio's default (based on the - current implementation of ``FlowControlMixin``). - - As soon as the HTTP request and response in the opening handshake are - processed: - - * the request path is available in the :attr:`path` attribute; - * the request and response HTTP headers are available in the - :attr:`request_headers` and :attr:`response_headers` attributes, - which are :class:`~websockets.http.Headers` instances. - - If a subprotocol was negotiated, it's available in the :attr:`subprotocol` - attribute. - - Once the connection is closed, the code is available in the - :attr:`close_code` attribute and the reason in :attr:`close_reason`. - - All these attributes must be treated as read-only. - - """ - - # There are only two differences between the client-side and server-side - # behavior: masking the payload and closing the underlying TCP connection. - # Set is_client = True/False and side = "client"/"server" to pick a side. - is_client: bool - side: str = "undefined" - - def __init__( - self, - *, - ping_interval: Optional[float] = 20, - ping_timeout: Optional[float] = 20, - close_timeout: Optional[float] = None, - max_size: Optional[int] = 2 ** 20, - max_queue: Optional[int] = 2 ** 5, - read_limit: int = 2 ** 16, - write_limit: int = 2 ** 16, - loop: Optional[asyncio.AbstractEventLoop] = None, - # The following arguments are kept only for backwards compatibility. - host: Optional[str] = None, - port: Optional[int] = None, - secure: Optional[bool] = None, - legacy_recv: bool = False, - timeout: Optional[float] = None, - ) -> None: - # Backwards compatibility: close_timeout used to be called timeout. - if timeout is None: - timeout = 10 - else: - warnings.warn("rename timeout to close_timeout", DeprecationWarning) - # If both are specified, timeout is ignored. - if close_timeout is None: - close_timeout = timeout - - self.ping_interval = ping_interval - self.ping_timeout = ping_timeout - self.close_timeout = close_timeout - self.max_size = max_size - self.max_queue = max_queue - self.read_limit = read_limit - self.write_limit = write_limit - - if loop is None: - loop = asyncio.get_event_loop() - self.loop = loop - - self._host = host - self._port = port - self._secure = secure - self.legacy_recv = legacy_recv - - # Configure read buffer limits. The high-water limit is defined by - # ``self.read_limit``. The ``limit`` argument controls the line length - # limit and half the buffer limit of :class:`~asyncio.StreamReader`. - # That's why it must be set to half of ``self.read_limit``. - self.reader = asyncio.StreamReader(limit=read_limit // 2, loop=loop) - - # Copied from asyncio.FlowControlMixin - self._paused = False - self._drain_waiter: Optional[asyncio.Future[None]] = None - - self._drain_lock = asyncio.Lock( - loop=loop if sys.version_info[:2] < (3, 8) else None - ) - - # This class implements the data transfer and closing handshake, which - # are shared between the client-side and the server-side. - # Subclasses implement the opening handshake and, on success, execute - # :meth:`connection_open` to change the state to OPEN. - self.state = State.CONNECTING - logger.debug("%s - state = CONNECTING", self.side) - - # HTTP protocol parameters. - self.path: str - self.request_headers: Headers - self.response_headers: Headers - - # WebSocket protocol parameters. - self.extensions: List[Extension] = [] - self.subprotocol: Optional[Subprotocol] = None - - # The close code and reason are set when receiving a close frame or - # losing the TCP connection. - self.close_code: int - self.close_reason: str - - # Completed when the connection state becomes CLOSED. Translates the - # :meth:`connection_lost` callback to a :class:`~asyncio.Future` - # that can be awaited. (Other :class:`~asyncio.Protocol` callbacks are - # translated by ``self.stream_reader``). - self.connection_lost_waiter: asyncio.Future[None] = loop.create_future() - - # Queue of received messages. - self.messages: Deque[Data] = collections.deque() - self._pop_message_waiter: Optional[asyncio.Future[None]] = None - self._put_message_waiter: Optional[asyncio.Future[None]] = None - - # Protect sending fragmented messages. - self._fragmented_message_waiter: Optional[asyncio.Future[None]] = None - - # Mapping of ping IDs to pong waiters, in chronological order. - self.pings: Dict[bytes, asyncio.Future[None]] = {} - - # Task running the data transfer. - self.transfer_data_task: asyncio.Task[None] - - # Exception that occurred during data transfer, if any. - self.transfer_data_exc: Optional[BaseException] = None - - # Task sending keepalive pings. - self.keepalive_ping_task: asyncio.Task[None] - - # Task closing the TCP connection. - self.close_connection_task: asyncio.Task[None] - - # Copied from asyncio.FlowControlMixin - async def _drain_helper(self) -> None: # pragma: no cover - if self.connection_lost_waiter.done(): - raise ConnectionResetError("Connection lost") - if not self._paused: - return - waiter = self._drain_waiter - assert waiter is None or waiter.cancelled() - waiter = self.loop.create_future() - self._drain_waiter = waiter - await waiter - - # Copied from asyncio.StreamWriter - async def _drain(self) -> None: # pragma: no cover - if self.reader is not None: - exc = self.reader.exception() - if exc is not None: - raise exc - if self.transport is not None: - if self.transport.is_closing(): - # Yield to the event loop so connection_lost() may be - # called. Without this, _drain_helper() would return - # immediately, and code that calls - # write(...); yield from drain() - # in a loop would never call connection_lost(), so it - # would not see an error when the socket is closed. - await asyncio.sleep( - 0, loop=self.loop if sys.version_info[:2] < (3, 8) else None - ) - await self._drain_helper() - - def connection_open(self) -> None: - """ - Callback when the WebSocket opening handshake completes. - - Enter the OPEN state and start the data transfer phase. - - """ - # 4.1. The WebSocket Connection is Established. - assert self.state is State.CONNECTING - self.state = State.OPEN - logger.debug("%s - state = OPEN", self.side) - # Start the task that receives incoming WebSocket messages. - self.transfer_data_task = self.loop.create_task(self.transfer_data()) - # Start the task that sends pings at regular intervals. - self.keepalive_ping_task = self.loop.create_task(self.keepalive_ping()) - # Start the task that eventually closes the TCP connection. - self.close_connection_task = self.loop.create_task(self.close_connection()) - - @property - def host(self) -> Optional[str]: - alternative = "remote_address" if self.is_client else "local_address" - warnings.warn(f"use {alternative}[0] instead of host", DeprecationWarning) - return self._host - - @property - def port(self) -> Optional[int]: - alternative = "remote_address" if self.is_client else "local_address" - warnings.warn(f"use {alternative}[1] instead of port", DeprecationWarning) - return self._port - - @property - def secure(self) -> Optional[bool]: - warnings.warn("don't use secure", DeprecationWarning) - return self._secure - - # Public API - - @property - def local_address(self) -> Any: - """ - Local address of the connection as a ``(host, port)`` tuple. - - When the connection isn't open, ``local_address`` is ``None``. - - """ - try: - transport = self.transport - except AttributeError: - return None - else: - return transport.get_extra_info("sockname") - - @property - def remote_address(self) -> Any: - """ - Remote address of the connection as a ``(host, port)`` tuple. - - When the connection isn't open, ``remote_address`` is ``None``. - - """ - try: - transport = self.transport - except AttributeError: - return None - else: - return transport.get_extra_info("peername") - - @property - def open(self) -> bool: - """ - ``True`` when the connection is usable. - - It may be used to detect disconnections. However, this approach is - discouraged per the EAFP_ principle. - - When ``open`` is ``False``, using the connection raises a - :exc:`~websockets.exceptions.ConnectionClosed` exception. - - .. _EAFP: https://docs.python.org/3/glossary.html#term-eafp - - """ - return self.state is State.OPEN and not self.transfer_data_task.done() - - @property - def closed(self) -> bool: - """ - ``True`` once the connection is closed. - - Be aware that both :attr:`open` and :attr:`closed` are ``False`` during - the opening and closing sequences. - - """ - return self.state is State.CLOSED - - async def wait_closed(self) -> None: - """ - Wait until the connection is closed. - - This is identical to :attr:`closed`, except it can be awaited. - - This can make it easier to handle connection termination, regardless - of its cause, in tasks that interact with the WebSocket connection. - - """ - await asyncio.shield(self.connection_lost_waiter) - - async def __aiter__(self) -> AsyncIterator[Data]: - """ - Iterate on received messages. - - Exit normally when the connection is closed with code 1000 or 1001. - - Raise an exception in other cases. - - """ - try: - while True: - yield await self.recv() - except ConnectionClosedOK: - return - - async def recv(self) -> Data: - """ - Receive the next message. - - Return a :class:`str` for a text frame and :class:`bytes` for a binary - frame. - - When the end of the message stream is reached, :meth:`recv` raises - :exc:`~websockets.exceptions.ConnectionClosed`. Specifically, it - raises :exc:`~websockets.exceptions.ConnectionClosedOK` after a normal - connection closure and - :exc:`~websockets.exceptions.ConnectionClosedError` after a protocol - error or a network failure. - - .. versionchanged:: 3.0 - - :meth:`recv` used to return ``None`` instead. Refer to the - changelog for details. - - Canceling :meth:`recv` is safe. There's no risk of losing the next - message. The next invocation of :meth:`recv` will return it. This - makes it possible to enforce a timeout by wrapping :meth:`recv` in - :func:`~asyncio.wait_for`. - - :raises ~websockets.exceptions.ConnectionClosed: when the - connection is closed - :raises RuntimeError: if two coroutines call :meth:`recv` concurrently - - """ - if self._pop_message_waiter is not None: - raise RuntimeError( - "cannot call recv while another coroutine " - "is already waiting for the next message" - ) - - # Don't await self.ensure_open() here: - # - messages could be available in the queue even if the connection - # is closed; - # - messages could be received before the closing frame even if the - # connection is closing. - - # Wait until there's a message in the queue (if necessary) or the - # connection is closed. - while len(self.messages) <= 0: - pop_message_waiter: asyncio.Future[None] = self.loop.create_future() - self._pop_message_waiter = pop_message_waiter - try: - # If asyncio.wait() is canceled, it doesn't cancel - # pop_message_waiter and self.transfer_data_task. - await asyncio.wait( - [pop_message_waiter, self.transfer_data_task], - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - return_when=asyncio.FIRST_COMPLETED, - ) - finally: - self._pop_message_waiter = None - - # If asyncio.wait(...) exited because self.transfer_data_task - # completed before receiving a new message, raise a suitable - # exception (or return None if legacy_recv is enabled). - if not pop_message_waiter.done(): - if self.legacy_recv: - return None # type: ignore - else: - # Wait until the connection is closed to raise - # ConnectionClosed with the correct code and reason. - await self.ensure_open() - - # Pop a message from the queue. - message = self.messages.popleft() - - # Notify transfer_data(). - if self._put_message_waiter is not None: - self._put_message_waiter.set_result(None) - self._put_message_waiter = None - - return message - - async def send( - self, message: Union[Data, Iterable[Data], AsyncIterable[Data]] - ) -> None: - """ - Send a message. - - A string (:class:`str`) is sent as a `Text frame`_. A bytestring or - bytes-like object (:class:`bytes`, :class:`bytearray`, or - :class:`memoryview`) is sent as a `Binary frame`_. - - .. _Text frame: https://tools.ietf.org/html/rfc6455#section-5.6 - .. _Binary frame: https://tools.ietf.org/html/rfc6455#section-5.6 - - :meth:`send` also accepts an iterable or an asynchronous iterable of - strings, bytestrings, or bytes-like objects. In that case the message - is fragmented. Each item is treated as a message fragment and sent in - its own frame. All items must be of the same type, or else - :meth:`send` will raise a :exc:`TypeError` and the connection will be - closed. - - :meth:`send` rejects dict-like objects because this is often an error. - If you wish to send the keys of a dict-like object as fragments, call - its :meth:`~dict.keys` method and pass the result to :meth:`send`. - - Canceling :meth:`send` is discouraged. Instead, you should close the - connection with :meth:`close`. Indeed, there only two situations where - :meth:`send` yields control to the event loop: - - 1. The write buffer is full. If you don't want to wait until enough - data is sent, your only alternative is to close the connection. - :meth:`close` will likely time out then abort the TCP connection. - 2. ``message`` is an asynchronous iterator. Stopping in the middle of - a fragmented message will cause a protocol error. Closing the - connection has the same effect. - - :raises TypeError: for unsupported inputs - - """ - await self.ensure_open() - - # While sending a fragmented message, prevent sending other messages - # until all fragments are sent. - while self._fragmented_message_waiter is not None: - await asyncio.shield(self._fragmented_message_waiter) - - # Unfragmented message -- this case must be handled first because - # strings and bytes-like objects are iterable. - - if isinstance(message, (str, bytes, bytearray, memoryview)): - opcode, data = prepare_data(message) - await self.write_frame(True, opcode, data) - - # Catch a common mistake -- passing a dict to send(). - - elif isinstance(message, Mapping): - raise TypeError("data is a dict-like object") - - # Fragmented message -- regular iterator. - - elif isinstance(message, Iterable): - - # Work around https://github.com/python/mypy/issues/6227 - message = cast(Iterable[Data], message) - - iter_message = iter(message) - try: - message_chunk = next(iter_message) - except StopIteration: - return - opcode, data = prepare_data(message_chunk) - - self._fragmented_message_waiter = asyncio.Future() - try: - # First fragment. - await self.write_frame(False, opcode, data) - - # Other fragments. - for message_chunk in iter_message: - confirm_opcode, data = prepare_data(message_chunk) - if confirm_opcode != opcode: - raise TypeError("data contains inconsistent types") - await self.write_frame(False, OP_CONT, data) - - # Final fragment. - await self.write_frame(True, OP_CONT, b"") - - except Exception: - # We're half-way through a fragmented message and we can't - # complete it. This makes the connection unusable. - self.fail_connection(1011) - raise - - finally: - self._fragmented_message_waiter.set_result(None) - self._fragmented_message_waiter = None - - # Fragmented message -- asynchronous iterator - - elif isinstance(message, AsyncIterable): - # aiter_message = aiter(message) without aiter - # https://github.com/python/mypy/issues/5738 - aiter_message = type(message).__aiter__(message) # type: ignore - try: - # message_chunk = anext(aiter_message) without anext - # https://github.com/python/mypy/issues/5738 - message_chunk = await type(aiter_message).__anext__( # type: ignore - aiter_message - ) - except StopAsyncIteration: - return - opcode, data = prepare_data(message_chunk) - - self._fragmented_message_waiter = asyncio.Future() - try: - # First fragment. - await self.write_frame(False, opcode, data) - - # Other fragments. - # https://github.com/python/mypy/issues/5738 - async for message_chunk in aiter_message: # type: ignore - confirm_opcode, data = prepare_data(message_chunk) - if confirm_opcode != opcode: - raise TypeError("data contains inconsistent types") - await self.write_frame(False, OP_CONT, data) - - # Final fragment. - await self.write_frame(True, OP_CONT, b"") - - except Exception: - # We're half-way through a fragmented message and we can't - # complete it. This makes the connection unusable. - self.fail_connection(1011) - raise - - finally: - self._fragmented_message_waiter.set_result(None) - self._fragmented_message_waiter = None - - else: - raise TypeError("data must be bytes, str, or iterable") - - async def close(self, code: int = 1000, reason: str = "") -> None: - """ - Perform the closing handshake. - - :meth:`close` waits for the other end to complete the handshake and - for the TCP connection to terminate. As a consequence, there's no need - to await :meth:`wait_closed`; :meth:`close` already does it. - - :meth:`close` is idempotent: it doesn't do anything once the - connection is closed. - - Wrapping :func:`close` in :func:`~asyncio.create_task` is safe, given - that errors during connection termination aren't particularly useful. - - Canceling :meth:`close` is discouraged. If it takes too long, you can - set a shorter ``close_timeout``. If you don't want to wait, let the - Python process exit, then the OS will close the TCP connection. - - :param code: WebSocket close code - :param reason: WebSocket close reason - - """ - try: - await asyncio.wait_for( - self.write_close_frame(serialize_close(code, reason)), - self.close_timeout, - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - ) - except asyncio.TimeoutError: - # If the close frame cannot be sent because the send buffers - # are full, the closing handshake won't complete anyway. - # Fail the connection to shut down faster. - self.fail_connection() - - # If no close frame is received within the timeout, wait_for() cancels - # the data transfer task and raises TimeoutError. - - # If close() is called multiple times concurrently and one of these - # calls hits the timeout, the data transfer task will be cancelled. - # Other calls will receive a CancelledError here. - - try: - # If close() is canceled during the wait, self.transfer_data_task - # is canceled before the timeout elapses. - await asyncio.wait_for( - self.transfer_data_task, - self.close_timeout, - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - ) - except (asyncio.TimeoutError, asyncio.CancelledError): - pass - - # Wait for the close connection task to close the TCP connection. - await asyncio.shield(self.close_connection_task) - - async def ping(self, data: Optional[Data] = None) -> Awaitable[None]: - """ - Send a ping. - - Return a :class:`~asyncio.Future` that will be completed when the - corresponding pong is received. You can ignore it if you don't intend - to wait. - - A ping may serve as a keepalive or as a check that the remote endpoint - received all messages up to this point:: - - pong_waiter = await ws.ping() - await pong_waiter # only if you want to wait for the pong - - By default, the ping contains four random bytes. This payload may be - overridden with the optional ``data`` argument which must be a string - (which will be encoded to UTF-8) or a bytes-like object. - - Canceling :meth:`ping` is discouraged. If :meth:`ping` doesn't return - immediately, it means the write buffer is full. If you don't want to - wait, you should close the connection. - - Canceling the :class:`~asyncio.Future` returned by :meth:`ping` has no - effect. - - """ - await self.ensure_open() - - if data is not None: - data = prepare_ctrl(data) - - # Protect against duplicates if a payload is explicitly set. - if data in self.pings: - raise ValueError("already waiting for a pong with the same data") - - # Generate a unique random payload otherwise. - while data is None or data in self.pings: - data = struct.pack("!I", random.getrandbits(32)) - - self.pings[data] = self.loop.create_future() - - await self.write_frame(True, OP_PING, data) - - return asyncio.shield(self.pings[data]) - - async def pong(self, data: Data = b"") -> None: - """ - Send a pong. - - An unsolicited pong may serve as a unidirectional heartbeat. - - The payload may be set with the optional ``data`` argument which must - be a string (which will be encoded to UTF-8) or a bytes-like object. - - Canceling :meth:`pong` is discouraged for the same reason as - :meth:`ping`. - - """ - await self.ensure_open() - - data = prepare_ctrl(data) - - await self.write_frame(True, OP_PONG, data) - - # Private methods - no guarantees. - - def connection_closed_exc(self) -> ConnectionClosed: - exception: ConnectionClosed - if self.close_code == 1000 or self.close_code == 1001: - exception = ConnectionClosedOK(self.close_code, self.close_reason) - else: - exception = ConnectionClosedError(self.close_code, self.close_reason) - # Chain to the exception that terminated data transfer, if any. - exception.__cause__ = self.transfer_data_exc - return exception - - async def ensure_open(self) -> None: - """ - Check that the WebSocket connection is open. - - Raise :exc:`~websockets.exceptions.ConnectionClosed` if it isn't. - - """ - # Handle cases from most common to least common for performance. - if self.state is State.OPEN: - # If self.transfer_data_task exited without a closing handshake, - # self.close_connection_task may be closing the connection, going - # straight from OPEN to CLOSED. - if self.transfer_data_task.done(): - await asyncio.shield(self.close_connection_task) - raise self.connection_closed_exc() - else: - return - - if self.state is State.CLOSED: - raise self.connection_closed_exc() - - if self.state is State.CLOSING: - # If we started the closing handshake, wait for its completion to - # get the proper close code and reason. self.close_connection_task - # will complete within 4 or 5 * close_timeout after close(). The - # CLOSING state also occurs when failing the connection. In that - # case self.close_connection_task will complete even faster. - await asyncio.shield(self.close_connection_task) - raise self.connection_closed_exc() - - # Control may only reach this point in buggy third-party subclasses. - assert self.state is State.CONNECTING - raise InvalidState("WebSocket connection isn't established yet") - - async def transfer_data(self) -> None: - """ - Read incoming messages and put them in a queue. - - This coroutine runs in a task until the closing handshake is started. - - """ - try: - while True: - message = await self.read_message() - - # Exit the loop when receiving a close frame. - if message is None: - break - - # Wait until there's room in the queue (if necessary). - if self.max_queue is not None: - while len(self.messages) >= self.max_queue: - self._put_message_waiter = self.loop.create_future() - try: - await asyncio.shield(self._put_message_waiter) - finally: - self._put_message_waiter = None - - # Put the message in the queue. - self.messages.append(message) - - # Notify recv(). - if self._pop_message_waiter is not None: - self._pop_message_waiter.set_result(None) - self._pop_message_waiter = None - - except asyncio.CancelledError as exc: - self.transfer_data_exc = exc - # If fail_connection() cancels this task, avoid logging the error - # twice and failing the connection again. - raise - - except ProtocolError as exc: - self.transfer_data_exc = exc - self.fail_connection(1002) - - except (ConnectionError, TimeoutError, EOFError) as exc: - # Reading data with self.reader.readexactly may raise: - # - most subclasses of ConnectionError if the TCP connection - # breaks, is reset, or is aborted; - # - TimeoutError if the TCP connection times out; - # - IncompleteReadError, a subclass of EOFError, if fewer - # bytes are available than requested. - self.transfer_data_exc = exc - self.fail_connection(1006) - - except UnicodeDecodeError as exc: - self.transfer_data_exc = exc - self.fail_connection(1007) - - except PayloadTooBig as exc: - self.transfer_data_exc = exc - self.fail_connection(1009) - - except Exception as exc: - # This shouldn't happen often because exceptions expected under - # regular circumstances are handled above. If it does, consider - # catching and handling more exceptions. - logger.error("Error in data transfer", exc_info=True) - - self.transfer_data_exc = exc - self.fail_connection(1011) - - async def read_message(self) -> Optional[Data]: - """ - Read a single message from the connection. - - Re-assemble data frames if the message is fragmented. - - Return ``None`` when the closing handshake is started. - - """ - frame = await self.read_data_frame(max_size=self.max_size) - - # A close frame was received. - if frame is None: - return None - - if frame.opcode == OP_TEXT: - text = True - elif frame.opcode == OP_BINARY: - text = False - else: # frame.opcode == OP_CONT - raise ProtocolError("unexpected opcode") - - # Shortcut for the common case - no fragmentation - if frame.fin: - return frame.data.decode("utf-8") if text else frame.data - - # 5.4. Fragmentation - chunks: List[Data] = [] - max_size = self.max_size - if text: - decoder_factory = codecs.getincrementaldecoder("utf-8") - decoder = decoder_factory(errors="strict") - if max_size is None: - - def append(frame: Frame) -> None: - nonlocal chunks - chunks.append(decoder.decode(frame.data, frame.fin)) - - else: - - def append(frame: Frame) -> None: - nonlocal chunks, max_size - chunks.append(decoder.decode(frame.data, frame.fin)) - assert isinstance(max_size, int) - max_size -= len(frame.data) - - else: - if max_size is None: - - def append(frame: Frame) -> None: - nonlocal chunks - chunks.append(frame.data) - - else: - - def append(frame: Frame) -> None: - nonlocal chunks, max_size - chunks.append(frame.data) - assert isinstance(max_size, int) - max_size -= len(frame.data) - - append(frame) - - while not frame.fin: - frame = await self.read_data_frame(max_size=max_size) - if frame is None: - raise ProtocolError("incomplete fragmented message") - if frame.opcode != OP_CONT: - raise ProtocolError("unexpected opcode") - append(frame) - - # mypy cannot figure out that chunks have the proper type. - return ("" if text else b"").join(chunks) # type: ignore - - async def read_data_frame(self, max_size: Optional[int]) -> Optional[Frame]: - """ - Read a single data frame from the connection. - - Process control frames received before the next data frame. - - Return ``None`` if a close frame is encountered before any data frame. - - """ - # 6.2. Receiving Data - while True: - frame = await self.read_frame(max_size) - - # 5.5. Control Frames - if frame.opcode == OP_CLOSE: - # 7.1.5. The WebSocket Connection Close Code - # 7.1.6. The WebSocket Connection Close Reason - self.close_code, self.close_reason = parse_close(frame.data) - try: - # Echo the original data instead of re-serializing it with - # serialize_close() because that fails when the close frame - # is empty and parse_close() synthetizes a 1005 close code. - await self.write_close_frame(frame.data) - except ConnectionClosed: - # It doesn't really matter if the connection was closed - # before we could send back a close frame. - pass - return None - - elif frame.opcode == OP_PING: - # Answer pings. - ping_hex = frame.data.hex() or "[empty]" - logger.debug( - "%s - received ping, sending pong: %s", self.side, ping_hex - ) - await self.pong(frame.data) - - elif frame.opcode == OP_PONG: - # Acknowledge pings on solicited pongs. - if frame.data in self.pings: - logger.debug( - "%s - received solicited pong: %s", - self.side, - frame.data.hex() or "[empty]", - ) - # Acknowledge all pings up to the one matching this pong. - ping_id = None - ping_ids = [] - for ping_id, ping in self.pings.items(): - ping_ids.append(ping_id) - if not ping.done(): - ping.set_result(None) - if ping_id == frame.data: - break - else: # pragma: no cover - assert False, "ping_id is in self.pings" - # Remove acknowledged pings from self.pings. - for ping_id in ping_ids: - del self.pings[ping_id] - ping_ids = ping_ids[:-1] - if ping_ids: - pings_hex = ", ".join( - ping_id.hex() or "[empty]" for ping_id in ping_ids - ) - plural = "s" if len(ping_ids) > 1 else "" - logger.debug( - "%s - acknowledged previous ping%s: %s", - self.side, - plural, - pings_hex, - ) - else: - logger.debug( - "%s - received unsolicited pong: %s", - self.side, - frame.data.hex() or "[empty]", - ) - - # 5.6. Data Frames - else: - return frame - - async def read_frame(self, max_size: Optional[int]) -> Frame: - """ - Read a single frame from the connection. - - """ - frame = await Frame.read( - self.reader.readexactly, - mask=not self.is_client, - max_size=max_size, - extensions=self.extensions, - ) - logger.debug("%s < %r", self.side, frame) - return frame - - async def write_frame( - self, fin: bool, opcode: int, data: bytes, *, _expected_state: int = State.OPEN - ) -> None: - # Defensive assertion for protocol compliance. - if self.state is not _expected_state: # pragma: no cover - raise InvalidState( - f"Cannot write to a WebSocket in the {self.state.name} state" - ) - - frame = Frame(fin, Opcode(opcode), data) - logger.debug("%s > %r", self.side, frame) - frame.write( - self.transport.write, mask=self.is_client, extensions=self.extensions - ) - - try: - # drain() cannot be called concurrently by multiple coroutines: - # http://bugs.python.org/issue29930. Remove this lock when no - # version of Python where this bugs exists is supported anymore. - async with self._drain_lock: - # Handle flow control automatically. - await self._drain() - except ConnectionError: - # Terminate the connection if the socket died. - self.fail_connection() - # Wait until the connection is closed to raise ConnectionClosed - # with the correct code and reason. - await self.ensure_open() - - async def write_close_frame(self, data: bytes = b"") -> None: - """ - Write a close frame if and only if the connection state is OPEN. - - This dedicated coroutine must be used for writing close frames to - ensure that at most one close frame is sent on a given connection. - - """ - # Test and set the connection state before sending the close frame to - # avoid sending two frames in case of concurrent calls. - if self.state is State.OPEN: - # 7.1.3. The WebSocket Closing Handshake is Started - self.state = State.CLOSING - logger.debug("%s - state = CLOSING", self.side) - - # 7.1.2. Start the WebSocket Closing Handshake - await self.write_frame(True, OP_CLOSE, data, _expected_state=State.CLOSING) - - async def keepalive_ping(self) -> None: - """ - Send a Ping frame and wait for a Pong frame at regular intervals. - - This coroutine exits when the connection terminates and one of the - following happens: - - - :meth:`ping` raises :exc:`ConnectionClosed`, or - - :meth:`close_connection` cancels :attr:`keepalive_ping_task`. - - """ - if self.ping_interval is None: - return - - try: - while True: - await asyncio.sleep( - self.ping_interval, - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - ) - - # ping() raises CancelledError if the connection is closed, - # when close_connection() cancels self.keepalive_ping_task. - - # ping() raises ConnectionClosed if the connection is lost, - # when connection_lost() calls abort_pings(). - - pong_waiter = await self.ping() - - if self.ping_timeout is not None: - try: - await asyncio.wait_for( - pong_waiter, - self.ping_timeout, - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - ) - except asyncio.TimeoutError: - logger.debug("%s ! timed out waiting for pong", self.side) - self.fail_connection(1011) - break - - # Remove this branch when dropping support for Python < 3.8 - # because CancelledError no longer inherits Exception. - except asyncio.CancelledError: - raise - - except ConnectionClosed: - pass - - except Exception: - logger.warning("Unexpected exception in keepalive ping task", exc_info=True) - - async def close_connection(self) -> None: - """ - 7.1.1. Close the WebSocket Connection - - When the opening handshake succeeds, :meth:`connection_open` starts - this coroutine in a task. It waits for the data transfer phase to - complete then it closes the TCP connection cleanly. - - When the opening handshake fails, :meth:`fail_connection` does the - same. There's no data transfer phase in that case. - - """ - try: - # Wait for the data transfer phase to complete. - if hasattr(self, "transfer_data_task"): - try: - await self.transfer_data_task - except asyncio.CancelledError: - pass - - # Cancel the keepalive ping task. - if hasattr(self, "keepalive_ping_task"): - self.keepalive_ping_task.cancel() - - # A client should wait for a TCP close from the server. - if self.is_client and hasattr(self, "transfer_data_task"): - if await self.wait_for_connection_lost(): - # Coverage marks this line as a partially executed branch. - # I supect a bug in coverage. Ignore it for now. - return # pragma: no cover - logger.debug("%s ! timed out waiting for TCP close", self.side) - - # Half-close the TCP connection if possible (when there's no TLS). - if self.transport.can_write_eof(): - logger.debug("%s x half-closing TCP connection", self.side) - self.transport.write_eof() - - if await self.wait_for_connection_lost(): - # Coverage marks this line as a partially executed branch. - # I supect a bug in coverage. Ignore it for now. - return # pragma: no cover - logger.debug("%s ! timed out waiting for TCP close", self.side) - - finally: - # The try/finally ensures that the transport never remains open, - # even if this coroutine is canceled (for example). - - # If connection_lost() was called, the TCP connection is closed. - # However, if TLS is enabled, the transport still needs closing. - # Else asyncio complains: ResourceWarning: unclosed transport. - if self.connection_lost_waiter.done() and self.transport.is_closing(): - return - - # Close the TCP connection. Buffers are flushed asynchronously. - logger.debug("%s x closing TCP connection", self.side) - self.transport.close() - - if await self.wait_for_connection_lost(): - return - logger.debug("%s ! timed out waiting for TCP close", self.side) - - # Abort the TCP connection. Buffers are discarded. - logger.debug("%s x aborting TCP connection", self.side) - self.transport.abort() - - # connection_lost() is called quickly after aborting. - # Coverage marks this line as a partially executed branch. - # I supect a bug in coverage. Ignore it for now. - await self.wait_for_connection_lost() # pragma: no cover - - async def wait_for_connection_lost(self) -> bool: - """ - Wait until the TCP connection is closed or ``self.close_timeout`` elapses. - - Return ``True`` if the connection is closed and ``False`` otherwise. - - """ - if not self.connection_lost_waiter.done(): - try: - await asyncio.wait_for( - asyncio.shield(self.connection_lost_waiter), - self.close_timeout, - loop=self.loop if sys.version_info[:2] < (3, 8) else None, - ) - except asyncio.TimeoutError: - pass - # Re-check self.connection_lost_waiter.done() synchronously because - # connection_lost() could run between the moment the timeout occurs - # and the moment this coroutine resumes running. - return self.connection_lost_waiter.done() - - def fail_connection(self, code: int = 1006, reason: str = "") -> None: - """ - 7.1.7. Fail the WebSocket Connection - - This requires: - - 1. Stopping all processing of incoming data, which means cancelling - :attr:`transfer_data_task`. The close code will be 1006 unless a - close frame was received earlier. - - 2. Sending a close frame with an appropriate code if the opening - handshake succeeded and the other side is likely to process it. - - 3. Closing the connection. :meth:`close_connection` takes care of - this once :attr:`transfer_data_task` exits after being canceled. - - (The specification describes these steps in the opposite order.) - - """ - logger.debug( - "%s ! failing %s WebSocket connection with code %d", - self.side, - self.state.name, - code, - ) - - # Cancel transfer_data_task if the opening handshake succeeded. - # cancel() is idempotent and ignored if the task is done already. - if hasattr(self, "transfer_data_task"): - self.transfer_data_task.cancel() - - # Send a close frame when the state is OPEN (a close frame was already - # sent if it's CLOSING), except when failing the connection because of - # an error reading from or writing to the network. - # Don't send a close frame if the connection is broken. - if code != 1006 and self.state is State.OPEN: - - frame_data = serialize_close(code, reason) - - # Write the close frame without draining the write buffer. - - # Keeping fail_connection() synchronous guarantees it can't - # get stuck and simplifies the implementation of the callers. - # Not drainig the write buffer is acceptable in this context. - - # This duplicates a few lines of code from write_close_frame() - # and write_frame(). - - self.state = State.CLOSING - logger.debug("%s - state = CLOSING", self.side) - - frame = Frame(True, OP_CLOSE, frame_data) - logger.debug("%s > %r", self.side, frame) - frame.write( - self.transport.write, mask=self.is_client, extensions=self.extensions - ) - - # Start close_connection_task if the opening handshake didn't succeed. - if not hasattr(self, "close_connection_task"): - self.close_connection_task = self.loop.create_task(self.close_connection()) - - def abort_pings(self) -> None: - """ - Raise ConnectionClosed in pending keepalive pings. - - They'll never receive a pong once the connection is closed. - - """ - assert self.state is State.CLOSED - exc = self.connection_closed_exc() - - for ping in self.pings.values(): - ping.set_exception(exc) - # If the exception is never retrieved, it will be logged when ping - # is garbage-collected. This is confusing for users. - # Given that ping is done (with an exception), canceling it does - # nothing, but it prevents logging the exception. - ping.cancel() - - if self.pings: - pings_hex = ", ".join(ping_id.hex() or "[empty]" for ping_id in self.pings) - plural = "s" if len(self.pings) > 1 else "" - logger.debug( - "%s - aborted pending ping%s: %s", self.side, plural, pings_hex - ) - - # asyncio.Protocol methods - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - """ - Configure write buffer limits. - - The high-water limit is defined by ``self.write_limit``. - - The low-water limit currently defaults to ``self.write_limit // 4`` in - :meth:`~asyncio.WriteTransport.set_write_buffer_limits`, which should - be all right for reasonable use cases of this library. - - This is the earliest point where we can get hold of the transport, - which means it's the best point for configuring it. - - """ - logger.debug("%s - event = connection_made(%s)", self.side, transport) - - transport = cast(asyncio.Transport, transport) - transport.set_write_buffer_limits(self.write_limit) - self.transport = transport - - # Copied from asyncio.StreamReaderProtocol - self.reader.set_transport(transport) - - def connection_lost(self, exc: Optional[Exception]) -> None: - """ - 7.1.4. The WebSocket Connection is Closed. - - """ - logger.debug("%s - event = connection_lost(%s)", self.side, exc) - self.state = State.CLOSED - logger.debug("%s - state = CLOSED", self.side) - if not hasattr(self, "close_code"): - self.close_code = 1006 - if not hasattr(self, "close_reason"): - self.close_reason = "" - logger.debug( - "%s x code = %d, reason = %s", - self.side, - self.close_code, - self.close_reason or "[no reason]", - ) - self.abort_pings() - # If self.connection_lost_waiter isn't pending, that's a bug, because: - # - it's set only here in connection_lost() which is called only once; - # - it must never be canceled. - self.connection_lost_waiter.set_result(None) - - if True: # pragma: no cover - - # Copied from asyncio.StreamReaderProtocol - if self.reader is not None: - if exc is None: - self.reader.feed_eof() - else: - self.reader.set_exception(exc) - - # Copied from asyncio.FlowControlMixin - # Wake up the writer if currently paused. - if not self._paused: - return - waiter = self._drain_waiter - if waiter is None: - return - self._drain_waiter = None - if waiter.done(): - return - if exc is None: - waiter.set_result(None) - else: - waiter.set_exception(exc) - - def pause_writing(self) -> None: # pragma: no cover - assert not self._paused - self._paused = True - - def resume_writing(self) -> None: # pragma: no cover - assert self._paused - self._paused = False - - waiter = self._drain_waiter - if waiter is not None: - self._drain_waiter = None - if not waiter.done(): - waiter.set_result(None) - - def data_received(self, data: bytes) -> None: - logger.debug("%s - event = data_received(<%d bytes>)", self.side, len(data)) - self.reader.feed_data(data) - - def eof_received(self) -> None: - """ - Close the transport after receiving EOF. - - The WebSocket protocol has its own closing handshake: endpoints close - the TCP or TLS connection after sending and receiving a close frame. - - As a consequence, they never need to write after receiving EOF, so - there's no reason to keep the transport open by returning ``True``. - - Besides, that doesn't work on TLS connections. - - """ - logger.debug("%s - event = eof_received()", self.side) - self.reader.feed_eof() +from .legacy.protocol import * # noqa diff --git a/src/websockets/server.py b/src/websockets/server.py index c2c818ce9..bd527be74 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -6,7 +6,6 @@ import logging from typing import Callable, Generator, List, Optional, Sequence, Tuple, Union, cast -from .asyncio_server import WebSocketServer, WebSocketServerProtocol, serve, unix_serve from .connection import CONNECTING, OPEN, SERVER, Connection from .datastructures import Headers, HeadersLike, MultipleValuesError from .exceptions import ( @@ -27,6 +26,12 @@ ) from .http import USER_AGENT from .http11 import Request, Response +from .legacy.server import ( # noqa + WebSocketServer, + WebSocketServerProtocol, + serve, + unix_serve, +) from .typing import ( ConnectionOption, ExtensionHeader, @@ -37,13 +42,7 @@ from .utils import accept_key -__all__ = [ - "serve", - "unix_serve", - "ServerConnection", - "WebSocketServerProtocol", - "WebSocketServer", -] +__all__ = ["ServerConnection"] logger = logging.getLogger(__name__) diff --git a/tests/__init__.py b/tests/__init__.py index 76c869f50..dd78609f5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,15 +1,5 @@ import logging -import warnings # Avoid displaying stack traces at the ERROR logging level. logging.basicConfig(level=logging.CRITICAL) - - -# Ignore deprecation warnings while refactoring is in progress -warnings.filterwarnings( - action="ignore", - message=r"websockets\.framing is deprecated", - category=DeprecationWarning, - module="websockets.framing", -) diff --git a/tests/legacy/__init__.py b/tests/legacy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/legacy/test_auth.py b/tests/legacy/test_auth.py new file mode 100644 index 000000000..bb8c6a6eb --- /dev/null +++ b/tests/legacy/test_auth.py @@ -0,0 +1,160 @@ +import unittest +import urllib.error + +from websockets.exceptions import InvalidStatusCode +from websockets.headers import build_authorization_basic +from websockets.legacy.auth import * +from websockets.legacy.auth import is_credentials + +from .test_client_server import ClientServerTestsMixin, with_client, with_server +from .utils import AsyncioTestCase + + +class AuthTests(unittest.TestCase): + def test_is_credentials(self): + self.assertTrue(is_credentials(("username", "password"))) + + def test_is_not_credentials(self): + self.assertFalse(is_credentials(None)) + self.assertFalse(is_credentials("username")) + + +class CustomWebSocketServerProtocol(BasicAuthWebSocketServerProtocol): + async def process_request(self, path, request_headers): + type(self).used = True + return await super().process_request(path, request_headers) + + +class AuthClientServerTests(ClientServerTestsMixin, AsyncioTestCase): + + create_protocol = basic_auth_protocol_factory( + realm="auth-tests", credentials=("hello", "iloveyou") + ) + + @with_server(create_protocol=create_protocol) + @with_client(user_info=("hello", "iloveyou")) + def test_basic_auth(self): + req_headers = self.client.request_headers + resp_headers = self.client.response_headers + self.assertEqual(req_headers["Authorization"], "Basic aGVsbG86aWxvdmV5b3U=") + self.assertNotIn("WWW-Authenticate", resp_headers) + + self.loop.run_until_complete(self.client.send("Hello!")) + self.loop.run_until_complete(self.client.recv()) + + def test_basic_auth_server_no_credentials(self): + with self.assertRaises(TypeError) as raised: + basic_auth_protocol_factory(realm="auth-tests", credentials=None) + self.assertEqual( + str(raised.exception), "provide either credentials or check_credentials" + ) + + def test_basic_auth_server_bad_credentials(self): + with self.assertRaises(TypeError) as raised: + basic_auth_protocol_factory(realm="auth-tests", credentials=42) + self.assertEqual(str(raised.exception), "invalid credentials argument: 42") + + create_protocol_multiple_credentials = basic_auth_protocol_factory( + realm="auth-tests", + credentials=[("hello", "iloveyou"), ("goodbye", "stillloveu")], + ) + + @with_server(create_protocol=create_protocol_multiple_credentials) + @with_client(user_info=("hello", "iloveyou")) + def test_basic_auth_server_multiple_credentials(self): + self.loop.run_until_complete(self.client.send("Hello!")) + self.loop.run_until_complete(self.client.recv()) + + def test_basic_auth_bad_multiple_credentials(self): + with self.assertRaises(TypeError) as raised: + basic_auth_protocol_factory( + realm="auth-tests", credentials=[("hello", "iloveyou"), 42] + ) + self.assertEqual( + str(raised.exception), + "invalid credentials argument: [('hello', 'iloveyou'), 42]", + ) + + async def check_credentials(username, password): + return password == "iloveyou" + + create_protocol_check_credentials = basic_auth_protocol_factory( + realm="auth-tests", + check_credentials=check_credentials, + ) + + @with_server(create_protocol=create_protocol_check_credentials) + @with_client(user_info=("hello", "iloveyou")) + def test_basic_auth_check_credentials(self): + self.loop.run_until_complete(self.client.send("Hello!")) + self.loop.run_until_complete(self.client.recv()) + + create_protocol_custom_protocol = basic_auth_protocol_factory( + realm="auth-tests", + credentials=[("hello", "iloveyou")], + create_protocol=CustomWebSocketServerProtocol, + ) + + @with_server(create_protocol=create_protocol_custom_protocol) + @with_client(user_info=("hello", "iloveyou")) + def test_basic_auth_custom_protocol(self): + self.assertTrue(CustomWebSocketServerProtocol.used) + del CustomWebSocketServerProtocol.used + self.loop.run_until_complete(self.client.send("Hello!")) + self.loop.run_until_complete(self.client.recv()) + + @with_server(create_protocol=create_protocol) + def test_basic_auth_missing_credentials(self): + with self.assertRaises(InvalidStatusCode) as raised: + self.start_client() + self.assertEqual(raised.exception.status_code, 401) + + @with_server(create_protocol=create_protocol) + def test_basic_auth_missing_credentials_details(self): + with self.assertRaises(urllib.error.HTTPError) as raised: + self.loop.run_until_complete(self.make_http_request()) + self.assertEqual(raised.exception.code, 401) + self.assertEqual( + raised.exception.headers["WWW-Authenticate"], + 'Basic realm="auth-tests", charset="UTF-8"', + ) + self.assertEqual(raised.exception.read().decode(), "Missing credentials\n") + + @with_server(create_protocol=create_protocol) + def test_basic_auth_unsupported_credentials(self): + with self.assertRaises(InvalidStatusCode) as raised: + self.start_client(extra_headers={"Authorization": "Digest ..."}) + self.assertEqual(raised.exception.status_code, 401) + + @with_server(create_protocol=create_protocol) + def test_basic_auth_unsupported_credentials_details(self): + with self.assertRaises(urllib.error.HTTPError) as raised: + self.loop.run_until_complete( + self.make_http_request(headers={"Authorization": "Digest ..."}) + ) + self.assertEqual(raised.exception.code, 401) + self.assertEqual( + raised.exception.headers["WWW-Authenticate"], + 'Basic realm="auth-tests", charset="UTF-8"', + ) + self.assertEqual(raised.exception.read().decode(), "Unsupported credentials\n") + + @with_server(create_protocol=create_protocol) + def test_basic_auth_invalid_credentials(self): + with self.assertRaises(InvalidStatusCode) as raised: + self.start_client(user_info=("hello", "ihateyou")) + self.assertEqual(raised.exception.status_code, 401) + + @with_server(create_protocol=create_protocol) + def test_basic_auth_invalid_credentials_details(self): + with self.assertRaises(urllib.error.HTTPError) as raised: + authorization = build_authorization_basic("hello", "ihateyou") + self.loop.run_until_complete( + self.make_http_request(headers={"Authorization": authorization}) + ) + self.assertEqual(raised.exception.code, 401) + self.assertEqual( + raised.exception.headers["WWW-Authenticate"], + 'Basic realm="auth-tests", charset="UTF-8"', + ) + self.assertEqual(raised.exception.read().decode(), "Invalid credentials\n") diff --git a/tests/test_asyncio_client_server.py b/tests/legacy/test_client_server.py similarity index 97% rename from tests/test_asyncio_client_server.py rename to tests/legacy/test_client_server.py index 76c29334e..499ea1d59 100644 --- a/tests/test_asyncio_client_server.py +++ b/tests/legacy/test_client_server.py @@ -13,8 +13,6 @@ import urllib.request import warnings -from websockets.asyncio_client import * -from websockets.asyncio_server import * from websockets.datastructures import Headers from websockets.exceptions import ( ConnectionClosed, @@ -28,19 +26,20 @@ PerMessageDeflate, ServerPerMessageDeflateFactory, ) -from websockets.handshake_legacy import build_response from websockets.http import USER_AGENT -from websockets.http_legacy import read_response -from websockets.protocol import State +from websockets.legacy.client import * +from websockets.legacy.handshake import build_response +from websockets.legacy.http import read_response +from websockets.legacy.protocol import State +from websockets.legacy.server import * from websockets.uri import parse_uri -from .extensions.test_base import ( +from ..extensions.test_base import ( ClientNoOpExtensionFactory, NoOpExtension, ServerNoOpExtensionFactory, ) -from .test_protocol import MS -from .utils import AsyncioTestCase +from .utils import MS, AsyncioTestCase # Generate TLS certificate with: @@ -49,7 +48,7 @@ # $ cat test_localhost.key test_localhost.crt > test_localhost.pem # $ rm test_localhost.key test_localhost.crt -testcert = bytes(pathlib.Path(__file__).with_name("test_localhost.pem")) +testcert = bytes(pathlib.Path(__file__).parent.with_name("test_localhost.pem")) async def handler(ws, path): @@ -1016,7 +1015,7 @@ def test_subprotocol_error_two_subprotocols(self, _process_subprotocol): self.run_loop_once() @with_server() - @unittest.mock.patch("websockets.asyncio_server.read_request") + @unittest.mock.patch("websockets.legacy.server.read_request") def test_server_receives_malformed_request(self, _read_request): _read_request.side_effect = ValueError("read_request failed") @@ -1024,7 +1023,7 @@ def test_server_receives_malformed_request(self, _read_request): self.start_client() @with_server() - @unittest.mock.patch("websockets.asyncio_client.read_response") + @unittest.mock.patch("websockets.legacy.client.read_response") def test_client_receives_malformed_response(self, _read_response): _read_response.side_effect = ValueError("read_response failed") @@ -1033,7 +1032,7 @@ def test_client_receives_malformed_response(self, _read_response): self.run_loop_once() @with_server() - @unittest.mock.patch("websockets.asyncio_client.build_request") + @unittest.mock.patch("websockets.legacy.client.build_request") def test_client_sends_invalid_handshake_request(self, _build_request): def wrong_build_request(headers): return "42" @@ -1044,7 +1043,7 @@ def wrong_build_request(headers): self.start_client() @with_server() - @unittest.mock.patch("websockets.asyncio_server.build_response") + @unittest.mock.patch("websockets.legacy.server.build_response") def test_server_sends_invalid_handshake_response(self, _build_response): def wrong_build_response(headers, key): return build_response(headers, "42") @@ -1055,7 +1054,7 @@ def wrong_build_response(headers, key): self.start_client() @with_server() - @unittest.mock.patch("websockets.asyncio_client.read_response") + @unittest.mock.patch("websockets.legacy.client.read_response") def test_server_does_not_switch_protocols(self, _read_response): async def wrong_read_response(stream): status_code, reason, headers = await read_response(stream) @@ -1069,7 +1068,7 @@ async def wrong_read_response(stream): @with_server() @unittest.mock.patch( - "websockets.asyncio_server.WebSocketServerProtocol.process_request" + "websockets.legacy.server.WebSocketServerProtocol.process_request" ) def test_server_error_in_handshake(self, _process_request): _process_request.side_effect = Exception("process_request crashed") @@ -1078,7 +1077,7 @@ def test_server_error_in_handshake(self, _process_request): self.start_client() @with_server() - @unittest.mock.patch("websockets.asyncio_server.WebSocketServerProtocol.send") + @unittest.mock.patch("websockets.legacy.server.WebSocketServerProtocol.send") def test_server_handler_crashes(self, send): send.side_effect = ValueError("send failed") @@ -1091,7 +1090,7 @@ def test_server_handler_crashes(self, send): self.assertEqual(self.client.close_code, 1011) @with_server() - @unittest.mock.patch("websockets.asyncio_server.WebSocketServerProtocol.close") + @unittest.mock.patch("websockets.legacy.server.WebSocketServerProtocol.close") def test_server_close_crashes(self, close): close.side_effect = ValueError("close failed") @@ -1164,10 +1163,10 @@ def test_invalid_status_error_during_client_connect(self): @with_server() @unittest.mock.patch( - "websockets.server.WebSocketServerProtocol.write_http_response" + "websockets.legacy.server.WebSocketServerProtocol.write_http_response" ) @unittest.mock.patch( - "websockets.asyncio_server.WebSocketServerProtocol.read_http_request" + "websockets.legacy.server.WebSocketServerProtocol.read_http_request" ) def test_connection_error_during_opening_handshake( self, _read_http_request, _write_http_response @@ -1186,7 +1185,7 @@ def test_connection_error_during_opening_handshake( _write_http_response.assert_not_called() @with_server() - @unittest.mock.patch("websockets.asyncio_server.WebSocketServerProtocol.close") + @unittest.mock.patch("websockets.legacy.server.WebSocketServerProtocol.close") def test_connection_error_during_closing_handshake(self, close): close.side_effect = ConnectionError diff --git a/tests/legacy/test_framing.py b/tests/legacy/test_framing.py new file mode 100644 index 000000000..ac870c79e --- /dev/null +++ b/tests/legacy/test_framing.py @@ -0,0 +1,171 @@ +import asyncio +import codecs +import unittest +import unittest.mock +import warnings + +from websockets.exceptions import PayloadTooBig, ProtocolError +from websockets.frames import OP_BINARY, OP_CLOSE, OP_PING, OP_PONG, OP_TEXT +from websockets.legacy.framing import * + +from .utils import AsyncioTestCase + + +class FramingTests(AsyncioTestCase): + def decode(self, message, mask=False, max_size=None, extensions=None): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(message) + stream.feed_eof() + with warnings.catch_warnings(record=True): + frame = self.loop.run_until_complete( + Frame.read( + stream.readexactly, + mask=mask, + max_size=max_size, + extensions=extensions, + ) + ) + # Make sure all the data was consumed. + self.assertTrue(stream.at_eof()) + return frame + + def encode(self, frame, mask=False, extensions=None): + write = unittest.mock.Mock() + with warnings.catch_warnings(record=True): + frame.write(write, mask=mask, extensions=extensions) + # Ensure the entire frame is sent with a single call to write(). + # Multiple calls cause TCP fragmentation and degrade performance. + self.assertEqual(write.call_count, 1) + # The frame data is the single positional argument of that call. + self.assertEqual(len(write.call_args[0]), 1) + self.assertEqual(len(write.call_args[1]), 0) + return write.call_args[0][0] + + def round_trip(self, message, expected, mask=False, extensions=None): + decoded = self.decode(message, mask, extensions=extensions) + self.assertEqual(decoded, expected) + encoded = self.encode(decoded, mask, extensions=extensions) + if mask: # non-deterministic encoding + decoded = self.decode(encoded, mask, extensions=extensions) + self.assertEqual(decoded, expected) + else: # deterministic encoding + self.assertEqual(encoded, message) + + def test_text(self): + self.round_trip(b"\x81\x04Spam", Frame(True, OP_TEXT, b"Spam")) + + def test_text_masked(self): + self.round_trip( + b"\x81\x84\x5b\xfb\xe1\xa8\x08\x8b\x80\xc5", + Frame(True, OP_TEXT, b"Spam"), + mask=True, + ) + + def test_binary(self): + self.round_trip(b"\x82\x04Eggs", Frame(True, OP_BINARY, b"Eggs")) + + def test_binary_masked(self): + self.round_trip( + b"\x82\x84\x53\xcd\xe2\x89\x16\xaa\x85\xfa", + Frame(True, OP_BINARY, b"Eggs"), + mask=True, + ) + + def test_non_ascii_text(self): + self.round_trip( + b"\x81\x05caf\xc3\xa9", Frame(True, OP_TEXT, "café".encode("utf-8")) + ) + + def test_non_ascii_text_masked(self): + self.round_trip( + b"\x81\x85\x64\xbe\xee\x7e\x07\xdf\x88\xbd\xcd", + Frame(True, OP_TEXT, "café".encode("utf-8")), + mask=True, + ) + + def test_close(self): + self.round_trip(b"\x88\x00", Frame(True, OP_CLOSE, b"")) + + def test_ping(self): + self.round_trip(b"\x89\x04ping", Frame(True, OP_PING, b"ping")) + + def test_pong(self): + self.round_trip(b"\x8a\x04pong", Frame(True, OP_PONG, b"pong")) + + def test_long(self): + self.round_trip( + b"\x82\x7e\x00\x7e" + 126 * b"a", Frame(True, OP_BINARY, 126 * b"a") + ) + + def test_very_long(self): + self.round_trip( + b"\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x00" + 65536 * b"a", + Frame(True, OP_BINARY, 65536 * b"a"), + ) + + def test_payload_too_big(self): + with self.assertRaises(PayloadTooBig): + self.decode(b"\x82\x7e\x04\x01" + 1025 * b"a", max_size=1024) + + def test_bad_reserved_bits(self): + for encoded in [b"\xc0\x00", b"\xa0\x00", b"\x90\x00"]: + with self.subTest(encoded=encoded): + with self.assertRaises(ProtocolError): + self.decode(encoded) + + def test_good_opcode(self): + for opcode in list(range(0x00, 0x03)) + list(range(0x08, 0x0B)): + encoded = bytes([0x80 | opcode, 0]) + with self.subTest(encoded=encoded): + self.decode(encoded) # does not raise an exception + + def test_bad_opcode(self): + for opcode in list(range(0x03, 0x08)) + list(range(0x0B, 0x10)): + encoded = bytes([0x80 | opcode, 0]) + with self.subTest(encoded=encoded): + with self.assertRaises(ProtocolError): + self.decode(encoded) + + def test_mask_flag(self): + # Mask flag correctly set. + self.decode(b"\x80\x80\x00\x00\x00\x00", mask=True) + # Mask flag incorrectly unset. + with self.assertRaises(ProtocolError): + self.decode(b"\x80\x80\x00\x00\x00\x00") + # Mask flag correctly unset. + self.decode(b"\x80\x00") + # Mask flag incorrectly set. + with self.assertRaises(ProtocolError): + self.decode(b"\x80\x00", mask=True) + + def test_control_frame_max_length(self): + # At maximum allowed length. + self.decode(b"\x88\x7e\x00\x7d" + 125 * b"a") + # Above maximum allowed length. + with self.assertRaises(ProtocolError): + self.decode(b"\x88\x7e\x00\x7e" + 126 * b"a") + + def test_fragmented_control_frame(self): + # Fin bit correctly set. + self.decode(b"\x88\x00") + # Fin bit incorrectly unset. + with self.assertRaises(ProtocolError): + self.decode(b"\x08\x00") + + def test_extensions(self): + class Rot13: + @staticmethod + def encode(frame): + assert frame.opcode == OP_TEXT + text = frame.data.decode() + data = codecs.encode(text, "rot13").encode() + return frame._replace(data=data) + + # This extensions is symmetrical. + @staticmethod + def decode(frame, *, max_size=None): + return Rot13.encode(frame) + + self.round_trip( + b"\x81\x05uryyb", Frame(True, OP_TEXT, b"hello"), extensions=[Rot13()] + ) diff --git a/tests/test_handshake_legacy.py b/tests/legacy/test_handshake.py similarity index 99% rename from tests/test_handshake_legacy.py rename to tests/legacy/test_handshake.py index c34b94e41..661ae64fc 100644 --- a/tests/test_handshake_legacy.py +++ b/tests/legacy/test_handshake.py @@ -8,7 +8,7 @@ InvalidHeaderValue, InvalidUpgrade, ) -from websockets.handshake_legacy import * +from websockets.legacy.handshake import * from websockets.utils import accept_key diff --git a/tests/test_http_legacy.py b/tests/legacy/test_http.py similarity index 98% rename from tests/test_http_legacy.py rename to tests/legacy/test_http.py index e4c75315e..5c9adc97f 100644 --- a/tests/test_http_legacy.py +++ b/tests/legacy/test_http.py @@ -1,8 +1,8 @@ import asyncio from websockets.exceptions import SecurityError -from websockets.http_legacy import * -from websockets.http_legacy import read_headers +from websockets.legacy.http import * +from websockets.legacy.http import read_headers from .utils import AsyncioTestCase diff --git a/tests/legacy/test_protocol.py b/tests/legacy/test_protocol.py new file mode 100644 index 000000000..218d05376 --- /dev/null +++ b/tests/legacy/test_protocol.py @@ -0,0 +1,1489 @@ +import asyncio +import contextlib +import sys +import unittest +import unittest.mock +import warnings + +from websockets.exceptions import ConnectionClosed, InvalidState +from websockets.frames import ( + OP_BINARY, + OP_CLOSE, + OP_CONT, + OP_PING, + OP_PONG, + OP_TEXT, + serialize_close, +) +from websockets.legacy.framing import Frame +from websockets.legacy.protocol import State, WebSocketCommonProtocol + +from .utils import MS, AsyncioTestCase + + +async def async_iterable(iterable): + for item in iterable: + yield item + + +class TransportMock(unittest.mock.Mock): + """ + Transport mock to control the protocol's inputs and outputs in tests. + + It calls the protocol's connection_made and connection_lost methods like + actual transports. + + It also calls the protocol's connection_open method to bypass the + WebSocket handshake. + + To simulate incoming data, tests call the protocol's data_received and + eof_received methods directly. + + They could also pause_writing and resume_writing to test flow control. + + """ + + # This should happen in __init__ but overriding Mock.__init__ is hard. + def setup_mock(self, loop, protocol): + self.loop = loop + self.protocol = protocol + self._eof = False + self._closing = False + # Simulate a successful TCP handshake. + self.protocol.connection_made(self) + # Simulate a successful WebSocket handshake. + self.protocol.connection_open() + + def can_write_eof(self): + return True + + def write_eof(self): + # When the protocol half-closes the TCP connection, it expects the + # other end to close it. Simulate that. + if not self._eof: + self.loop.call_soon(self.close) + self._eof = True + + def close(self): + # Simulate how actual transports drop the connection. + if not self._closing: + self.loop.call_soon(self.protocol.connection_lost, None) + self._closing = True + + def abort(self): + # Change this to an `if` if tests call abort() multiple times. + assert self.protocol.state is not State.CLOSED + self.loop.call_soon(self.protocol.connection_lost, None) + + +class CommonTests: + """ + Mixin that defines most tests but doesn't inherit unittest.TestCase. + + Tests are run by the ServerTests and ClientTests subclasses. + + """ + + def setUp(self): + super().setUp() + # Disable pings to make it easier to test what frames are sent exactly. + self.protocol = WebSocketCommonProtocol(ping_interval=None) + self.transport = TransportMock() + self.transport.setup_mock(self.loop, self.protocol) + + def tearDown(self): + self.transport.close() + self.loop.run_until_complete(self.protocol.close()) + super().tearDown() + + # Utilities for writing tests. + + def make_drain_slow(self, delay=MS): + # Process connection_made in order to initialize self.protocol.transport. + self.run_loop_once() + + original_drain = self.protocol._drain + + async def delayed_drain(): + await asyncio.sleep( + delay, loop=self.loop if sys.version_info[:2] < (3, 8) else None + ) + await original_drain() + + self.protocol._drain = delayed_drain + + close_frame = Frame(True, OP_CLOSE, serialize_close(1000, "close")) + local_close = Frame(True, OP_CLOSE, serialize_close(1000, "local")) + remote_close = Frame(True, OP_CLOSE, serialize_close(1000, "remote")) + + def receive_frame(self, frame): + """ + Make the protocol receive a frame. + + """ + write = self.protocol.data_received + mask = not self.protocol.is_client + frame.write(write, mask=mask) + + def receive_eof(self): + """ + Make the protocol receive the end of the data stream. + + Since ``WebSocketCommonProtocol.eof_received`` returns ``None``, an + actual transport would close itself after calling it. This function + emulates that behavior. + + """ + self.protocol.eof_received() + self.loop.call_soon(self.transport.close) + + def receive_eof_if_client(self): + """ + Like receive_eof, but only if this is the client side. + + Since the server is supposed to initiate the termination of the TCP + connection, this method helps making tests work for both sides. + + """ + if self.protocol.is_client: + self.receive_eof() + + def close_connection(self, code=1000, reason="close"): + """ + Execute a closing handshake. + + This puts the connection in the CLOSED state. + + """ + close_frame_data = serialize_close(code, reason) + # Prepare the response to the closing handshake from the remote side. + self.receive_frame(Frame(True, OP_CLOSE, close_frame_data)) + self.receive_eof_if_client() + # Trigger the closing handshake from the local side and complete it. + self.loop.run_until_complete(self.protocol.close(code, reason)) + # Empty the outgoing data stream so we can make assertions later on. + self.assertOneFrameSent(True, OP_CLOSE, close_frame_data) + + assert self.protocol.state is State.CLOSED + + def half_close_connection_local(self, code=1000, reason="close"): + """ + Start a closing handshake but do not complete it. + + The main difference with `close_connection` is that the connection is + left in the CLOSING state until the event loop runs again. + + The current implementation returns a task that must be awaited or + canceled, else asyncio complains about destroying a pending task. + + """ + close_frame_data = serialize_close(code, reason) + # Trigger the closing handshake from the local endpoint. + close_task = self.loop.create_task(self.protocol.close(code, reason)) + self.run_loop_once() # wait_for executes + self.run_loop_once() # write_frame executes + # Empty the outgoing data stream so we can make assertions later on. + self.assertOneFrameSent(True, OP_CLOSE, close_frame_data) + + assert self.protocol.state is State.CLOSING + + # Complete the closing sequence at 1ms intervals so the test can run + # at each point even it goes back to the event loop several times. + self.loop.call_later( + MS, self.receive_frame, Frame(True, OP_CLOSE, close_frame_data) + ) + self.loop.call_later(2 * MS, self.receive_eof_if_client) + + # This task must be awaited or canceled by the caller. + return close_task + + def half_close_connection_remote(self, code=1000, reason="close"): + """ + Receive a closing handshake but do not complete it. + + The main difference with `close_connection` is that the connection is + left in the CLOSING state until the event loop runs again. + + """ + # On the server side, websockets completes the closing handshake and + # closes the TCP connection immediately. Yield to the event loop after + # sending the close frame to run the test while the connection is in + # the CLOSING state. + if not self.protocol.is_client: + self.make_drain_slow() + + close_frame_data = serialize_close(code, reason) + # Trigger the closing handshake from the remote endpoint. + self.receive_frame(Frame(True, OP_CLOSE, close_frame_data)) + self.run_loop_once() # read_frame executes + # Empty the outgoing data stream so we can make assertions later on. + self.assertOneFrameSent(True, OP_CLOSE, close_frame_data) + + assert self.protocol.state is State.CLOSING + + # Complete the closing sequence at 1ms intervals so the test can run + # at each point even it goes back to the event loop several times. + self.loop.call_later(2 * MS, self.receive_eof_if_client) + + def process_invalid_frames(self): + """ + Make the protocol fail quickly after simulating invalid data. + + To achieve this, this function triggers the protocol's eof_received, + which interrupts pending reads waiting for more data. + + """ + self.run_loop_once() + self.receive_eof() + self.loop.run_until_complete(self.protocol.close_connection_task) + + def sent_frames(self): + """ + Read all frames sent to the transport. + + """ + stream = asyncio.StreamReader(loop=self.loop) + + for (data,), kw in self.transport.write.call_args_list: + stream.feed_data(data) + self.transport.write.call_args_list = [] + stream.feed_eof() + + frames = [] + while not stream.at_eof(): + frames.append( + self.loop.run_until_complete( + Frame.read(stream.readexactly, mask=self.protocol.is_client) + ) + ) + return frames + + def last_sent_frame(self): + """ + Read the last frame sent to the transport. + + This method assumes that at most one frame was sent. It raises an + AssertionError otherwise. + + """ + frames = self.sent_frames() + if frames: + assert len(frames) == 1 + return frames[0] + + def assertFramesSent(self, *frames): + self.assertEqual(self.sent_frames(), [Frame(*args) for args in frames]) + + def assertOneFrameSent(self, *args): + self.assertEqual(self.last_sent_frame(), Frame(*args)) + + def assertNoFrameSent(self): + self.assertIsNone(self.last_sent_frame()) + + def assertConnectionClosed(self, code, message): + # The following line guarantees that connection_lost was called. + self.assertEqual(self.protocol.state, State.CLOSED) + # A close frame was received. + self.assertEqual(self.protocol.close_code, code) + self.assertEqual(self.protocol.close_reason, message) + + def assertConnectionFailed(self, code, message): + # The following line guarantees that connection_lost was called. + self.assertEqual(self.protocol.state, State.CLOSED) + # No close frame was received. + self.assertEqual(self.protocol.close_code, 1006) + self.assertEqual(self.protocol.close_reason, "") + # A close frame was sent -- unless the connection was already lost. + if code == 1006: + self.assertNoFrameSent() + else: + self.assertOneFrameSent(True, OP_CLOSE, serialize_close(code, message)) + + @contextlib.contextmanager + def assertCompletesWithin(self, min_time, max_time): + t0 = self.loop.time() + yield + t1 = self.loop.time() + dt = t1 - t0 + self.assertGreaterEqual(dt, min_time, f"Too fast: {dt} < {min_time}") + self.assertLess(dt, max_time, f"Too slow: {dt} >= {max_time}") + + # Test constructor. + + def test_timeout_backwards_compatibility(self): + with warnings.catch_warnings(record=True) as recorded_warnings: + protocol = WebSocketCommonProtocol(timeout=5) + + self.assertEqual(protocol.close_timeout, 5) + + self.assertEqual(len(recorded_warnings), 1) + warning = recorded_warnings[0].message + self.assertEqual(str(warning), "rename timeout to close_timeout") + self.assertEqual(type(warning), DeprecationWarning) + + # Test public attributes. + + def test_local_address(self): + get_extra_info = unittest.mock.Mock(return_value=("host", 4312)) + self.transport.get_extra_info = get_extra_info + + self.assertEqual(self.protocol.local_address, ("host", 4312)) + get_extra_info.assert_called_with("sockname") + + def test_local_address_before_connection(self): + # Emulate the situation before connection_open() runs. + _transport = self.protocol.transport + del self.protocol.transport + try: + self.assertEqual(self.protocol.local_address, None) + finally: + self.protocol.transport = _transport + + def test_remote_address(self): + get_extra_info = unittest.mock.Mock(return_value=("host", 4312)) + self.transport.get_extra_info = get_extra_info + + self.assertEqual(self.protocol.remote_address, ("host", 4312)) + get_extra_info.assert_called_with("peername") + + def test_remote_address_before_connection(self): + # Emulate the situation before connection_open() runs. + _transport = self.protocol.transport + del self.protocol.transport + try: + self.assertEqual(self.protocol.remote_address, None) + finally: + self.protocol.transport = _transport + + def test_open(self): + self.assertTrue(self.protocol.open) + self.close_connection() + self.assertFalse(self.protocol.open) + + def test_closed(self): + self.assertFalse(self.protocol.closed) + self.close_connection() + self.assertTrue(self.protocol.closed) + + def test_wait_closed(self): + wait_closed = self.loop.create_task(self.protocol.wait_closed()) + self.assertFalse(wait_closed.done()) + self.close_connection() + self.assertTrue(wait_closed.done()) + + # Test the recv coroutine. + + def test_recv_text(self): + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, "café") + + def test_recv_binary(self): + self.receive_frame(Frame(True, OP_BINARY, b"tea")) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, b"tea") + + def test_recv_on_closing_connection_local(self): + close_task = self.half_close_connection_local() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.recv()) + + self.loop.run_until_complete(close_task) # cleanup + + def test_recv_on_closing_connection_remote(self): + self.half_close_connection_remote() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.recv()) + + def test_recv_on_closed_connection(self): + self.close_connection() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.recv()) + + def test_recv_protocol_error(self): + self.receive_frame(Frame(True, OP_CONT, "café".encode("utf-8"))) + self.process_invalid_frames() + self.assertConnectionFailed(1002, "") + + def test_recv_unicode_error(self): + self.receive_frame(Frame(True, OP_TEXT, "café".encode("latin-1"))) + self.process_invalid_frames() + self.assertConnectionFailed(1007, "") + + def test_recv_text_payload_too_big(self): + self.protocol.max_size = 1024 + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8") * 205)) + self.process_invalid_frames() + self.assertConnectionFailed(1009, "") + + def test_recv_binary_payload_too_big(self): + self.protocol.max_size = 1024 + self.receive_frame(Frame(True, OP_BINARY, b"tea" * 342)) + self.process_invalid_frames() + self.assertConnectionFailed(1009, "") + + def test_recv_text_no_max_size(self): + self.protocol.max_size = None # for test coverage + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8") * 205)) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, "café" * 205) + + def test_recv_binary_no_max_size(self): + self.protocol.max_size = None # for test coverage + self.receive_frame(Frame(True, OP_BINARY, b"tea" * 342)) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, b"tea" * 342) + + def test_recv_queue_empty(self): + recv = self.loop.create_task(self.protocol.recv()) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete( + asyncio.wait_for(asyncio.shield(recv), timeout=MS) + ) + + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) + data = self.loop.run_until_complete(recv) + self.assertEqual(data, "café") + + def test_recv_queue_full(self): + self.protocol.max_queue = 2 + # Test internals because it's hard to verify buffers from the outside. + self.assertEqual(list(self.protocol.messages), []) + + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) + self.run_loop_once() + self.assertEqual(list(self.protocol.messages), ["café"]) + + self.receive_frame(Frame(True, OP_BINARY, b"tea")) + self.run_loop_once() + self.assertEqual(list(self.protocol.messages), ["café", b"tea"]) + + self.receive_frame(Frame(True, OP_BINARY, b"milk")) + self.run_loop_once() + self.assertEqual(list(self.protocol.messages), ["café", b"tea"]) + + self.loop.run_until_complete(self.protocol.recv()) + self.run_loop_once() + self.assertEqual(list(self.protocol.messages), [b"tea", b"milk"]) + + self.loop.run_until_complete(self.protocol.recv()) + self.run_loop_once() + self.assertEqual(list(self.protocol.messages), [b"milk"]) + + self.loop.run_until_complete(self.protocol.recv()) + self.run_loop_once() + self.assertEqual(list(self.protocol.messages), []) + + def test_recv_queue_no_limit(self): + self.protocol.max_queue = None + + for _ in range(100): + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) + self.run_loop_once() + + # Incoming message queue can contain at least 100 messages. + self.assertEqual(list(self.protocol.messages), ["café"] * 100) + + for _ in range(100): + self.loop.run_until_complete(self.protocol.recv()) + + self.assertEqual(list(self.protocol.messages), []) + + def test_recv_other_error(self): + async def read_message(): + raise Exception("BOOM") + + self.protocol.read_message = read_message + self.process_invalid_frames() + self.assertConnectionFailed(1011, "") + + def test_recv_canceled(self): + recv = self.loop.create_task(self.protocol.recv()) + self.loop.call_soon(recv.cancel) + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(recv) + + # The next frame doesn't disappear in a vacuum (it used to). + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, "café") + + def test_recv_canceled_race_condition(self): + recv = self.loop.create_task( + asyncio.wait_for(self.protocol.recv(), timeout=0.000_001) + ) + self.loop.call_soon( + self.receive_frame, Frame(True, OP_TEXT, "café".encode("utf-8")) + ) + + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(recv) + + # The previous frame doesn't disappear in a vacuum (it used to). + self.receive_frame(Frame(True, OP_TEXT, "tea".encode("utf-8"))) + data = self.loop.run_until_complete(self.protocol.recv()) + # If we're getting "tea" there, it means "café" was swallowed (ha, ha). + self.assertEqual(data, "café") + + def test_recv_when_transfer_data_cancelled(self): + # Clog incoming queue. + self.protocol.max_queue = 1 + self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) + self.receive_frame(Frame(True, OP_BINARY, b"tea")) + self.run_loop_once() + + # Flow control kicks in (check with an implementation detail). + self.assertFalse(self.protocol._put_message_waiter.done()) + + # Schedule recv(). + recv = self.loop.create_task(self.protocol.recv()) + + # Cancel transfer_data_task (again, implementation detail). + self.protocol.fail_connection() + self.run_loop_once() + self.assertTrue(self.protocol.transfer_data_task.cancelled()) + + # recv() completes properly. + self.assertEqual(self.loop.run_until_complete(recv), "café") + + def test_recv_prevents_concurrent_calls(self): + recv = self.loop.create_task(self.protocol.recv()) + + with self.assertRaises(RuntimeError) as raised: + self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual( + str(raised.exception), + "cannot call recv while another coroutine " + "is already waiting for the next message", + ) + recv.cancel() + + # Test the send coroutine. + + def test_send_text(self): + self.loop.run_until_complete(self.protocol.send("café")) + self.assertOneFrameSent(True, OP_TEXT, "café".encode("utf-8")) + + def test_send_binary(self): + self.loop.run_until_complete(self.protocol.send(b"tea")) + self.assertOneFrameSent(True, OP_BINARY, b"tea") + + def test_send_binary_from_bytearray(self): + self.loop.run_until_complete(self.protocol.send(bytearray(b"tea"))) + self.assertOneFrameSent(True, OP_BINARY, b"tea") + + def test_send_binary_from_memoryview(self): + self.loop.run_until_complete(self.protocol.send(memoryview(b"tea"))) + self.assertOneFrameSent(True, OP_BINARY, b"tea") + + def test_send_binary_from_non_contiguous_memoryview(self): + self.loop.run_until_complete(self.protocol.send(memoryview(b"tteeaa")[::2])) + self.assertOneFrameSent(True, OP_BINARY, b"tea") + + def test_send_dict(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.send({"not": "encoded"})) + self.assertNoFrameSent() + + def test_send_type_error(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.send(42)) + self.assertNoFrameSent() + + def test_send_iterable_text(self): + self.loop.run_until_complete(self.protocol.send(["ca", "fé"])) + self.assertFramesSent( + (False, OP_TEXT, "ca".encode("utf-8")), + (False, OP_CONT, "fé".encode("utf-8")), + (True, OP_CONT, "".encode("utf-8")), + ) + + def test_send_iterable_binary(self): + self.loop.run_until_complete(self.protocol.send([b"te", b"a"])) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_iterable_binary_from_bytearray(self): + self.loop.run_until_complete( + self.protocol.send([bytearray(b"te"), bytearray(b"a")]) + ) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_iterable_binary_from_memoryview(self): + self.loop.run_until_complete( + self.protocol.send([memoryview(b"te"), memoryview(b"a")]) + ) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_iterable_binary_from_non_contiguous_memoryview(self): + self.loop.run_until_complete( + self.protocol.send([memoryview(b"ttee")[::2], memoryview(b"aa")[::2]]) + ) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_empty_iterable(self): + self.loop.run_until_complete(self.protocol.send([])) + self.assertNoFrameSent() + + def test_send_iterable_type_error(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.send([42])) + self.assertNoFrameSent() + + def test_send_iterable_mixed_type_error(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.send(["café", b"tea"])) + self.assertFramesSent( + (False, OP_TEXT, "café".encode("utf-8")), + (True, OP_CLOSE, serialize_close(1011, "")), + ) + + def test_send_iterable_prevents_concurrent_send(self): + self.make_drain_slow(2 * MS) + + async def send_iterable(): + await self.protocol.send(["ca", "fé"]) + + async def send_concurrent(): + await asyncio.sleep(MS) + await self.protocol.send(b"tea") + + self.loop.run_until_complete(asyncio.gather(send_iterable(), send_concurrent())) + self.assertFramesSent( + (False, OP_TEXT, "ca".encode("utf-8")), + (False, OP_CONT, "fé".encode("utf-8")), + (True, OP_CONT, "".encode("utf-8")), + (True, OP_BINARY, b"tea"), + ) + + def test_send_async_iterable_text(self): + self.loop.run_until_complete(self.protocol.send(async_iterable(["ca", "fé"]))) + self.assertFramesSent( + (False, OP_TEXT, "ca".encode("utf-8")), + (False, OP_CONT, "fé".encode("utf-8")), + (True, OP_CONT, "".encode("utf-8")), + ) + + def test_send_async_iterable_binary(self): + self.loop.run_until_complete(self.protocol.send(async_iterable([b"te", b"a"]))) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_async_iterable_binary_from_bytearray(self): + self.loop.run_until_complete( + self.protocol.send(async_iterable([bytearray(b"te"), bytearray(b"a")])) + ) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_async_iterable_binary_from_memoryview(self): + self.loop.run_until_complete( + self.protocol.send(async_iterable([memoryview(b"te"), memoryview(b"a")])) + ) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_async_iterable_binary_from_non_contiguous_memoryview(self): + self.loop.run_until_complete( + self.protocol.send( + async_iterable([memoryview(b"ttee")[::2], memoryview(b"aa")[::2]]) + ) + ) + self.assertFramesSent( + (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") + ) + + def test_send_empty_async_iterable(self): + self.loop.run_until_complete(self.protocol.send(async_iterable([]))) + self.assertNoFrameSent() + + def test_send_async_iterable_type_error(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.send(async_iterable([42]))) + self.assertNoFrameSent() + + def test_send_async_iterable_mixed_type_error(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete( + self.protocol.send(async_iterable(["café", b"tea"])) + ) + self.assertFramesSent( + (False, OP_TEXT, "café".encode("utf-8")), + (True, OP_CLOSE, serialize_close(1011, "")), + ) + + def test_send_async_iterable_prevents_concurrent_send(self): + self.make_drain_slow(2 * MS) + + async def send_async_iterable(): + await self.protocol.send(async_iterable(["ca", "fé"])) + + async def send_concurrent(): + await asyncio.sleep(MS) + await self.protocol.send(b"tea") + + self.loop.run_until_complete( + asyncio.gather(send_async_iterable(), send_concurrent()) + ) + self.assertFramesSent( + (False, OP_TEXT, "ca".encode("utf-8")), + (False, OP_CONT, "fé".encode("utf-8")), + (True, OP_CONT, "".encode("utf-8")), + (True, OP_BINARY, b"tea"), + ) + + def test_send_on_closing_connection_local(self): + close_task = self.half_close_connection_local() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.send("foobar")) + + self.assertNoFrameSent() + + self.loop.run_until_complete(close_task) # cleanup + + def test_send_on_closing_connection_remote(self): + self.half_close_connection_remote() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.send("foobar")) + + self.assertNoFrameSent() + + def test_send_on_closed_connection(self): + self.close_connection() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.send("foobar")) + + self.assertNoFrameSent() + + # Test the ping coroutine. + + def test_ping_default(self): + self.loop.run_until_complete(self.protocol.ping()) + # With our testing tools, it's more convenient to extract the expected + # ping data from the library's internals than from the frame sent. + ping_data = next(iter(self.protocol.pings)) + self.assertIsInstance(ping_data, bytes) + self.assertEqual(len(ping_data), 4) + self.assertOneFrameSent(True, OP_PING, ping_data) + + def test_ping_text(self): + self.loop.run_until_complete(self.protocol.ping("café")) + self.assertOneFrameSent(True, OP_PING, "café".encode("utf-8")) + + def test_ping_binary(self): + self.loop.run_until_complete(self.protocol.ping(b"tea")) + self.assertOneFrameSent(True, OP_PING, b"tea") + + def test_ping_binary_from_bytearray(self): + self.loop.run_until_complete(self.protocol.ping(bytearray(b"tea"))) + self.assertOneFrameSent(True, OP_PING, b"tea") + + def test_ping_binary_from_memoryview(self): + self.loop.run_until_complete(self.protocol.ping(memoryview(b"tea"))) + self.assertOneFrameSent(True, OP_PING, b"tea") + + def test_ping_binary_from_non_contiguous_memoryview(self): + self.loop.run_until_complete(self.protocol.ping(memoryview(b"tteeaa")[::2])) + self.assertOneFrameSent(True, OP_PING, b"tea") + + def test_ping_type_error(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.ping(42)) + self.assertNoFrameSent() + + def test_ping_on_closing_connection_local(self): + close_task = self.half_close_connection_local() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.ping()) + + self.assertNoFrameSent() + + self.loop.run_until_complete(close_task) # cleanup + + def test_ping_on_closing_connection_remote(self): + self.half_close_connection_remote() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.ping()) + + self.assertNoFrameSent() + + def test_ping_on_closed_connection(self): + self.close_connection() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.ping()) + + self.assertNoFrameSent() + + # Test the pong coroutine. + + def test_pong_default(self): + self.loop.run_until_complete(self.protocol.pong()) + self.assertOneFrameSent(True, OP_PONG, b"") + + def test_pong_text(self): + self.loop.run_until_complete(self.protocol.pong("café")) + self.assertOneFrameSent(True, OP_PONG, "café".encode("utf-8")) + + def test_pong_binary(self): + self.loop.run_until_complete(self.protocol.pong(b"tea")) + self.assertOneFrameSent(True, OP_PONG, b"tea") + + def test_pong_binary_from_bytearray(self): + self.loop.run_until_complete(self.protocol.pong(bytearray(b"tea"))) + self.assertOneFrameSent(True, OP_PONG, b"tea") + + def test_pong_binary_from_memoryview(self): + self.loop.run_until_complete(self.protocol.pong(memoryview(b"tea"))) + self.assertOneFrameSent(True, OP_PONG, b"tea") + + def test_pong_binary_from_non_contiguous_memoryview(self): + self.loop.run_until_complete(self.protocol.pong(memoryview(b"tteeaa")[::2])) + self.assertOneFrameSent(True, OP_PONG, b"tea") + + def test_pong_type_error(self): + with self.assertRaises(TypeError): + self.loop.run_until_complete(self.protocol.pong(42)) + self.assertNoFrameSent() + + def test_pong_on_closing_connection_local(self): + close_task = self.half_close_connection_local() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.pong()) + + self.assertNoFrameSent() + + self.loop.run_until_complete(close_task) # cleanup + + def test_pong_on_closing_connection_remote(self): + self.half_close_connection_remote() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.pong()) + + self.assertNoFrameSent() + + def test_pong_on_closed_connection(self): + self.close_connection() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.pong()) + + self.assertNoFrameSent() + + # Test the protocol's logic for acknowledging pings with pongs. + + def test_answer_ping(self): + self.receive_frame(Frame(True, OP_PING, b"test")) + self.run_loop_once() + self.assertOneFrameSent(True, OP_PONG, b"test") + + def test_ignore_pong(self): + self.receive_frame(Frame(True, OP_PONG, b"test")) + self.run_loop_once() + self.assertNoFrameSent() + + def test_acknowledge_ping(self): + ping = self.loop.run_until_complete(self.protocol.ping()) + self.assertFalse(ping.done()) + ping_frame = self.last_sent_frame() + pong_frame = Frame(True, OP_PONG, ping_frame.data) + self.receive_frame(pong_frame) + self.run_loop_once() + self.run_loop_once() + self.assertTrue(ping.done()) + + def test_abort_ping(self): + ping = self.loop.run_until_complete(self.protocol.ping()) + # Remove the frame from the buffer, else close_connection() complains. + self.last_sent_frame() + self.assertFalse(ping.done()) + self.close_connection() + self.assertTrue(ping.done()) + self.assertIsInstance(ping.exception(), ConnectionClosed) + + def test_abort_ping_does_not_log_exception_if_not_retreived(self): + self.loop.run_until_complete(self.protocol.ping()) + # Get the internal Future, which isn't directly returned by ping(). + (ping,) = self.protocol.pings.values() + # Remove the frame from the buffer, else close_connection() complains. + self.last_sent_frame() + self.close_connection() + # Check a private attribute, for lack of a better solution. + self.assertFalse(ping._log_traceback) + + def test_acknowledge_previous_pings(self): + pings = [ + (self.loop.run_until_complete(self.protocol.ping()), self.last_sent_frame()) + for i in range(3) + ] + # Unsolicited pong doesn't acknowledge pings + self.receive_frame(Frame(True, OP_PONG, b"")) + self.run_loop_once() + self.run_loop_once() + self.assertFalse(pings[0][0].done()) + self.assertFalse(pings[1][0].done()) + self.assertFalse(pings[2][0].done()) + # Pong acknowledges all previous pings + self.receive_frame(Frame(True, OP_PONG, pings[1][1].data)) + self.run_loop_once() + self.run_loop_once() + self.assertTrue(pings[0][0].done()) + self.assertTrue(pings[1][0].done()) + self.assertFalse(pings[2][0].done()) + + def test_acknowledge_aborted_ping(self): + ping = self.loop.run_until_complete(self.protocol.ping()) + ping_frame = self.last_sent_frame() + # Clog incoming queue. This lets connection_lost() abort pending pings + # with a ConnectionClosed exception before transfer_data_task + # terminates and close_connection cancels keepalive_ping_task. + self.protocol.max_queue = 1 + self.receive_frame(Frame(True, OP_TEXT, b"1")) + self.receive_frame(Frame(True, OP_TEXT, b"2")) + # Add pong frame to the queue. + pong_frame = Frame(True, OP_PONG, ping_frame.data) + self.receive_frame(pong_frame) + # Connection drops. + self.receive_eof() + self.loop.run_until_complete(self.protocol.wait_closed()) + # Ping receives a ConnectionClosed exception. + with self.assertRaises(ConnectionClosed): + ping.result() + + # transfer_data doesn't crash, which would be logged. + with self.assertNoLogs(): + # Unclog incoming queue. + self.loop.run_until_complete(self.protocol.recv()) + self.loop.run_until_complete(self.protocol.recv()) + + def test_canceled_ping(self): + ping = self.loop.run_until_complete(self.protocol.ping()) + ping_frame = self.last_sent_frame() + ping.cancel() + pong_frame = Frame(True, OP_PONG, ping_frame.data) + self.receive_frame(pong_frame) + self.run_loop_once() + self.run_loop_once() + self.assertTrue(ping.cancelled()) + + def test_duplicate_ping(self): + self.loop.run_until_complete(self.protocol.ping(b"foobar")) + self.assertOneFrameSent(True, OP_PING, b"foobar") + with self.assertRaises(ValueError): + self.loop.run_until_complete(self.protocol.ping(b"foobar")) + self.assertNoFrameSent() + + # Test the protocol's logic for rebuilding fragmented messages. + + def test_fragmented_text(self): + self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) + self.receive_frame(Frame(True, OP_CONT, "fé".encode("utf-8"))) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, "café") + + def test_fragmented_binary(self): + self.receive_frame(Frame(False, OP_BINARY, b"t")) + self.receive_frame(Frame(False, OP_CONT, b"e")) + self.receive_frame(Frame(True, OP_CONT, b"a")) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, b"tea") + + def test_fragmented_text_payload_too_big(self): + self.protocol.max_size = 1024 + self.receive_frame(Frame(False, OP_TEXT, "café".encode("utf-8") * 100)) + self.receive_frame(Frame(True, OP_CONT, "café".encode("utf-8") * 105)) + self.process_invalid_frames() + self.assertConnectionFailed(1009, "") + + def test_fragmented_binary_payload_too_big(self): + self.protocol.max_size = 1024 + self.receive_frame(Frame(False, OP_BINARY, b"tea" * 171)) + self.receive_frame(Frame(True, OP_CONT, b"tea" * 171)) + self.process_invalid_frames() + self.assertConnectionFailed(1009, "") + + def test_fragmented_text_no_max_size(self): + self.protocol.max_size = None # for test coverage + self.receive_frame(Frame(False, OP_TEXT, "café".encode("utf-8") * 100)) + self.receive_frame(Frame(True, OP_CONT, "café".encode("utf-8") * 105)) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, "café" * 205) + + def test_fragmented_binary_no_max_size(self): + self.protocol.max_size = None # for test coverage + self.receive_frame(Frame(False, OP_BINARY, b"tea" * 171)) + self.receive_frame(Frame(True, OP_CONT, b"tea" * 171)) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, b"tea" * 342) + + def test_control_frame_within_fragmented_text(self): + self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) + self.receive_frame(Frame(True, OP_PING, b"")) + self.receive_frame(Frame(True, OP_CONT, "fé".encode("utf-8"))) + data = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(data, "café") + self.assertOneFrameSent(True, OP_PONG, b"") + + def test_unterminated_fragmented_text(self): + self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) + # Missing the second part of the fragmented frame. + self.receive_frame(Frame(True, OP_BINARY, b"tea")) + self.process_invalid_frames() + self.assertConnectionFailed(1002, "") + + def test_close_handshake_in_fragmented_text(self): + self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) + self.receive_frame(Frame(True, OP_CLOSE, b"")) + self.process_invalid_frames() + # The RFC may have overlooked this case: it says that control frames + # can be interjected in the middle of a fragmented message and that a + # close frame must be echoed. Even though there's an unterminated + # message, technically, the closing handshake was successful. + self.assertConnectionClosed(1005, "") + + def test_connection_close_in_fragmented_text(self): + self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) + self.process_invalid_frames() + self.assertConnectionFailed(1006, "") + + # Test miscellaneous code paths to ensure full coverage. + + def test_connection_lost(self): + # Test calling connection_lost without going through close_connection. + self.protocol.connection_lost(None) + + self.assertConnectionFailed(1006, "") + + def test_ensure_open_before_opening_handshake(self): + # Simulate a bug by forcibly reverting the protocol state. + self.protocol.state = State.CONNECTING + + with self.assertRaises(InvalidState): + self.loop.run_until_complete(self.protocol.ensure_open()) + + def test_ensure_open_during_unclean_close(self): + # Process connection_made in order to start transfer_data_task. + self.run_loop_once() + + # Ensure the test terminates quickly. + self.loop.call_later(MS, self.receive_eof_if_client) + + # Simulate the case when close() times out sending a close frame. + self.protocol.fail_connection() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.ensure_open()) + + def test_legacy_recv(self): + # By default legacy_recv in disabled. + self.assertEqual(self.protocol.legacy_recv, False) + + self.close_connection() + + # Enable legacy_recv. + self.protocol.legacy_recv = True + + # Now recv() returns None instead of raising ConnectionClosed. + self.assertIsNone(self.loop.run_until_complete(self.protocol.recv())) + + def test_connection_closed_attributes(self): + self.close_connection() + + with self.assertRaises(ConnectionClosed) as context: + self.loop.run_until_complete(self.protocol.recv()) + + connection_closed_exc = context.exception + self.assertEqual(connection_closed_exc.code, 1000) + self.assertEqual(connection_closed_exc.reason, "close") + + # Test the protocol logic for sending keepalive pings. + + def restart_protocol_with_keepalive_ping( + self, ping_interval=3 * MS, ping_timeout=3 * MS + ): + initial_protocol = self.protocol + # copied from tearDown + self.transport.close() + self.loop.run_until_complete(self.protocol.close()) + # copied from setUp, but enables keepalive pings + self.protocol = WebSocketCommonProtocol( + ping_interval=ping_interval, ping_timeout=ping_timeout + ) + self.transport = TransportMock() + self.transport.setup_mock(self.loop, self.protocol) + self.protocol.is_client = initial_protocol.is_client + self.protocol.side = initial_protocol.side + + def test_keepalive_ping(self): + self.restart_protocol_with_keepalive_ping() + + # Ping is sent at 3ms and acknowledged at 4ms. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + (ping_1,) = tuple(self.protocol.pings) + self.assertOneFrameSent(True, OP_PING, ping_1) + self.receive_frame(Frame(True, OP_PONG, ping_1)) + + # Next ping is sent at 7ms. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + (ping_2,) = tuple(self.protocol.pings) + self.assertOneFrameSent(True, OP_PING, ping_2) + + # The keepalive ping task goes on. + self.assertFalse(self.protocol.keepalive_ping_task.done()) + + def test_keepalive_ping_not_acknowledged_closes_connection(self): + self.restart_protocol_with_keepalive_ping() + + # Ping is sent at 3ms and not acknowleged. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + (ping_1,) = tuple(self.protocol.pings) + self.assertOneFrameSent(True, OP_PING, ping_1) + + # Connection is closed at 6ms. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + self.assertOneFrameSent(True, OP_CLOSE, serialize_close(1011, "")) + + # The keepalive ping task is complete. + self.assertEqual(self.protocol.keepalive_ping_task.result(), None) + + def test_keepalive_ping_stops_when_connection_closing(self): + self.restart_protocol_with_keepalive_ping() + close_task = self.half_close_connection_local() + + # No ping sent at 3ms because the closing handshake is in progress. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + self.assertNoFrameSent() + + # The keepalive ping task terminated. + self.assertTrue(self.protocol.keepalive_ping_task.cancelled()) + + self.loop.run_until_complete(close_task) # cleanup + + def test_keepalive_ping_stops_when_connection_closed(self): + self.restart_protocol_with_keepalive_ping() + self.close_connection() + + # The keepalive ping task terminated. + self.assertTrue(self.protocol.keepalive_ping_task.cancelled()) + + def test_keepalive_ping_does_not_crash_when_connection_lost(self): + self.restart_protocol_with_keepalive_ping() + # Clog incoming queue. This lets connection_lost() abort pending pings + # with a ConnectionClosed exception before transfer_data_task + # terminates and close_connection cancels keepalive_ping_task. + self.protocol.max_queue = 1 + self.receive_frame(Frame(True, OP_TEXT, b"1")) + self.receive_frame(Frame(True, OP_TEXT, b"2")) + # Ping is sent at 3ms. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + (ping_waiter,) = tuple(self.protocol.pings.values()) + # Connection drops. + self.receive_eof() + self.loop.run_until_complete(self.protocol.wait_closed()) + + # The ping waiter receives a ConnectionClosed exception. + with self.assertRaises(ConnectionClosed): + ping_waiter.result() + # The keepalive ping task terminated properly. + self.assertIsNone(self.protocol.keepalive_ping_task.result()) + + # Unclog incoming queue to terminate the test quickly. + self.loop.run_until_complete(self.protocol.recv()) + self.loop.run_until_complete(self.protocol.recv()) + + def test_keepalive_ping_with_no_ping_interval(self): + self.restart_protocol_with_keepalive_ping(ping_interval=None) + + # No ping is sent at 3ms. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + self.assertNoFrameSent() + + def test_keepalive_ping_with_no_ping_timeout(self): + self.restart_protocol_with_keepalive_ping(ping_timeout=None) + + # Ping is sent at 3ms and not acknowleged. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + (ping_1,) = tuple(self.protocol.pings) + self.assertOneFrameSent(True, OP_PING, ping_1) + + # Next ping is sent at 7ms anyway. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + ping_1_again, ping_2 = tuple(self.protocol.pings) + self.assertEqual(ping_1, ping_1_again) + self.assertOneFrameSent(True, OP_PING, ping_2) + + # The keepalive ping task goes on. + self.assertFalse(self.protocol.keepalive_ping_task.done()) + + def test_keepalive_ping_unexpected_error(self): + self.restart_protocol_with_keepalive_ping() + + async def ping(): + raise Exception("BOOM") + + self.protocol.ping = ping + + # The keepalive ping task fails when sending a ping at 3ms. + self.loop.run_until_complete(asyncio.sleep(4 * MS)) + + # The keepalive ping task is complete. + # It logs and swallows the exception. + self.assertEqual(self.protocol.keepalive_ping_task.result(), None) + + # Test the protocol logic for closing the connection. + + def test_local_close(self): + # Emulate how the remote endpoint answers the closing handshake. + self.loop.call_later(MS, self.receive_frame, self.close_frame) + self.loop.call_later(MS, self.receive_eof_if_client) + + # Run the closing handshake. + self.loop.run_until_complete(self.protocol.close(reason="close")) + + self.assertConnectionClosed(1000, "close") + self.assertOneFrameSent(*self.close_frame) + + # Closing the connection again is a no-op. + self.loop.run_until_complete(self.protocol.close(reason="oh noes!")) + + self.assertConnectionClosed(1000, "close") + self.assertNoFrameSent() + + def test_remote_close(self): + # Emulate how the remote endpoint initiates the closing handshake. + self.loop.call_later(MS, self.receive_frame, self.close_frame) + self.loop.call_later(MS, self.receive_eof_if_client) + + # Wait for some data in order to process the handshake. + # After recv() raises ConnectionClosed, the connection is closed. + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(self.protocol.recv()) + + self.assertConnectionClosed(1000, "close") + self.assertOneFrameSent(*self.close_frame) + + # Closing the connection again is a no-op. + self.loop.run_until_complete(self.protocol.close(reason="oh noes!")) + + self.assertConnectionClosed(1000, "close") + self.assertNoFrameSent() + + def test_remote_close_and_connection_lost(self): + self.make_drain_slow() + # Drop the connection right after receiving a close frame, + # which prevents echoing the close frame properly. + self.receive_frame(self.close_frame) + self.receive_eof() + + with self.assertNoLogs(): + self.loop.run_until_complete(self.protocol.close(reason="oh noes!")) + + self.assertConnectionClosed(1000, "close") + self.assertOneFrameSent(*self.close_frame) + + def test_simultaneous_close(self): + # Receive the incoming close frame right after self.protocol.close() + # starts executing. This reproduces the error described in: + # https://github.com/aaugustin/websockets/issues/339 + self.loop.call_soon(self.receive_frame, self.remote_close) + self.loop.call_soon(self.receive_eof_if_client) + + self.loop.run_until_complete(self.protocol.close(reason="local")) + + self.assertConnectionClosed(1000, "remote") + # The current implementation sends a close frame in response to the + # close frame received from the remote end. It skips the close frame + # that should be sent as a result of calling close(). + self.assertOneFrameSent(*self.remote_close) + + def test_close_preserves_incoming_frames(self): + self.receive_frame(Frame(True, OP_TEXT, b"hello")) + + self.loop.call_later(MS, self.receive_frame, self.close_frame) + self.loop.call_later(MS, self.receive_eof_if_client) + self.loop.run_until_complete(self.protocol.close(reason="close")) + + self.assertConnectionClosed(1000, "close") + self.assertOneFrameSent(*self.close_frame) + + next_message = self.loop.run_until_complete(self.protocol.recv()) + self.assertEqual(next_message, "hello") + + def test_close_protocol_error(self): + invalid_close_frame = Frame(True, OP_CLOSE, b"\x00") + self.receive_frame(invalid_close_frame) + self.receive_eof_if_client() + self.run_loop_once() + self.loop.run_until_complete(self.protocol.close(reason="close")) + + self.assertConnectionFailed(1002, "") + + def test_close_connection_lost(self): + self.receive_eof() + self.run_loop_once() + self.loop.run_until_complete(self.protocol.close(reason="close")) + + self.assertConnectionFailed(1006, "") + + def test_local_close_during_recv(self): + recv = self.loop.create_task(self.protocol.recv()) + + self.loop.call_later(MS, self.receive_frame, self.close_frame) + self.loop.call_later(MS, self.receive_eof_if_client) + + self.loop.run_until_complete(self.protocol.close(reason="close")) + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(recv) + + self.assertConnectionClosed(1000, "close") + + # There is no test_remote_close_during_recv because it would be identical + # to test_remote_close. + + def test_remote_close_during_send(self): + self.make_drain_slow() + send = self.loop.create_task(self.protocol.send("hello")) + + self.receive_frame(self.close_frame) + self.receive_eof() + + with self.assertRaises(ConnectionClosed): + self.loop.run_until_complete(send) + + self.assertConnectionClosed(1000, "close") + + # There is no test_local_close_during_send because this cannot really + # happen, considering that writes are serialized. + + +class ServerTests(CommonTests, AsyncioTestCase): + def setUp(self): + super().setUp() + self.protocol.is_client = False + self.protocol.side = "server" + + def test_local_close_send_close_frame_timeout(self): + self.protocol.close_timeout = 10 * MS + self.make_drain_slow(50 * MS) + # If we can't send a close frame, time out in 10ms. + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(9 * MS, 19 * MS): + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1006, "") + + def test_local_close_receive_close_frame_timeout(self): + self.protocol.close_timeout = 10 * MS + # If the client doesn't send a close frame, time out in 10ms. + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(9 * MS, 19 * MS): + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1006, "") + + def test_local_close_connection_lost_timeout_after_write_eof(self): + self.protocol.close_timeout = 10 * MS + # If the client doesn't close its side of the TCP connection after we + # half-close our side with write_eof(), time out in 10ms. + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(9 * MS, 19 * MS): + # HACK: disable write_eof => other end drops connection emulation. + self.transport._eof = True + self.receive_frame(self.close_frame) + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1000, "close") + + def test_local_close_connection_lost_timeout_after_close(self): + self.protocol.close_timeout = 10 * MS + # If the client doesn't close its side of the TCP connection after we + # half-close our side with write_eof() and close it with close(), time + # out in 20ms. + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(19 * MS, 29 * MS): + # HACK: disable write_eof => other end drops connection emulation. + self.transport._eof = True + # HACK: disable close => other end drops connection emulation. + self.transport._closing = True + self.receive_frame(self.close_frame) + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1000, "close") + + +class ClientTests(CommonTests, AsyncioTestCase): + def setUp(self): + super().setUp() + self.protocol.is_client = True + self.protocol.side = "client" + + def test_local_close_send_close_frame_timeout(self): + self.protocol.close_timeout = 10 * MS + self.make_drain_slow(50 * MS) + # If we can't send a close frame, time out in 20ms. + # - 10ms waiting for sending a close frame + # - 10ms waiting for receiving a half-close + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(19 * MS, 29 * MS): + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1006, "") + + def test_local_close_receive_close_frame_timeout(self): + self.protocol.close_timeout = 10 * MS + # If the server doesn't send a close frame, time out in 20ms: + # - 10ms waiting for receiving a close frame + # - 10ms waiting for receiving a half-close + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(19 * MS, 29 * MS): + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1006, "") + + def test_local_close_connection_lost_timeout_after_write_eof(self): + self.protocol.close_timeout = 10 * MS + # If the server doesn't half-close its side of the TCP connection + # after we send a close frame, time out in 20ms: + # - 10ms waiting for receiving a half-close + # - 10ms waiting for receiving a close after write_eof + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(19 * MS, 29 * MS): + # HACK: disable write_eof => other end drops connection emulation. + self.transport._eof = True + self.receive_frame(self.close_frame) + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1000, "close") + + def test_local_close_connection_lost_timeout_after_close(self): + self.protocol.close_timeout = 10 * MS + # If the client doesn't close its side of the TCP connection after we + # half-close our side with write_eof() and close it with close(), time + # out in 20ms. + # - 10ms waiting for receiving a half-close + # - 10ms waiting for receiving a close after write_eof + # - 10ms waiting for receiving a close after close + # Check the timing within -1/+9ms for robustness. + with self.assertCompletesWithin(29 * MS, 39 * MS): + # HACK: disable write_eof => other end drops connection emulation. + self.transport._eof = True + # HACK: disable close => other end drops connection emulation. + self.transport._closing = True + self.receive_frame(self.close_frame) + self.loop.run_until_complete(self.protocol.close(reason="close")) + self.assertConnectionClosed(1000, "close") diff --git a/tests/legacy/utils.py b/tests/legacy/utils.py new file mode 100644 index 000000000..983a91edf --- /dev/null +++ b/tests/legacy/utils.py @@ -0,0 +1,93 @@ +import asyncio +import contextlib +import functools +import logging +import os +import time +import unittest + + +class AsyncioTestCase(unittest.TestCase): + """ + Base class for tests that sets up an isolated event loop for each test. + + """ + + def __init_subclass__(cls, **kwargs): + """ + Convert test coroutines to test functions. + + This supports asychronous tests transparently. + + """ + super().__init_subclass__(**kwargs) + for name in unittest.defaultTestLoader.getTestCaseNames(cls): + test = getattr(cls, name) + if asyncio.iscoroutinefunction(test): + setattr(cls, name, cls.convert_async_to_sync(test)) + + @staticmethod + def convert_async_to_sync(test): + """ + Convert a test coroutine to a test function. + + """ + + @functools.wraps(test) + def test_func(self, *args, **kwargs): + return self.loop.run_until_complete(test(self, *args, **kwargs)) + + return test_func + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + super().tearDown() + + def run_loop_once(self): + # Process callbacks scheduled with call_soon by appending a callback + # to stop the event loop then running it until it hits that callback. + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + + @contextlib.contextmanager + def assertNoLogs(self, logger="websockets", level=logging.ERROR): + """ + No message is logged on the given logger with at least the given level. + + """ + with self.assertLogs(logger, level) as logs: + # We want to test that no log message is emitted + # but assertLogs expects at least one log message. + logging.getLogger(logger).log(level, "dummy") + yield + + level_name = logging.getLevelName(level) + self.assertEqual(logs.output, [f"{level_name}:{logger}:dummy"]) + + def assertDeprecationWarnings(self, recorded_warnings, expected_warnings): + """ + Check recorded deprecation warnings match a list of expected messages. + + """ + self.assertEqual(len(recorded_warnings), len(expected_warnings)) + for recorded, expected in zip(recorded_warnings, expected_warnings): + actual = recorded.message + self.assertEqual(str(actual), expected) + self.assertEqual(type(actual), DeprecationWarning) + + +# Unit for timeouts. May be increased on slow machines by setting the +# WEBSOCKETS_TESTS_TIMEOUT_FACTOR environment variable. +MS = 0.001 * int(os.environ.get("WEBSOCKETS_TESTS_TIMEOUT_FACTOR", 1)) + +# asyncio's debug mode has a 10x performance penalty for this test suite. +if os.environ.get("PYTHONASYNCIODEBUG"): # pragma: no cover + MS *= 10 + +# Ensure that timeouts are larger than the clock's resolution (for Windows). +MS = max(MS, 2.5 * time.get_clock_info("monotonic").resolution) diff --git a/tests/test_auth.py b/tests/test_auth.py index ce23f913d..01ca207c7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,160 +1,2 @@ -import unittest -import urllib.error - -from websockets.auth import * -from websockets.auth import is_credentials -from websockets.exceptions import InvalidStatusCode -from websockets.headers import build_authorization_basic - -from .test_asyncio_client_server import ClientServerTestsMixin, with_client, with_server -from .utils import AsyncioTestCase - - -class AuthTests(unittest.TestCase): - def test_is_credentials(self): - self.assertTrue(is_credentials(("username", "password"))) - - def test_is_not_credentials(self): - self.assertFalse(is_credentials(None)) - self.assertFalse(is_credentials("username")) - - -class CustomWebSocketServerProtocol(BasicAuthWebSocketServerProtocol): - async def process_request(self, path, request_headers): - type(self).used = True - return await super().process_request(path, request_headers) - - -class AuthClientServerTests(ClientServerTestsMixin, AsyncioTestCase): - - create_protocol = basic_auth_protocol_factory( - realm="auth-tests", credentials=("hello", "iloveyou") - ) - - @with_server(create_protocol=create_protocol) - @with_client(user_info=("hello", "iloveyou")) - def test_basic_auth(self): - req_headers = self.client.request_headers - resp_headers = self.client.response_headers - self.assertEqual(req_headers["Authorization"], "Basic aGVsbG86aWxvdmV5b3U=") - self.assertNotIn("WWW-Authenticate", resp_headers) - - self.loop.run_until_complete(self.client.send("Hello!")) - self.loop.run_until_complete(self.client.recv()) - - def test_basic_auth_server_no_credentials(self): - with self.assertRaises(TypeError) as raised: - basic_auth_protocol_factory(realm="auth-tests", credentials=None) - self.assertEqual( - str(raised.exception), "provide either credentials or check_credentials" - ) - - def test_basic_auth_server_bad_credentials(self): - with self.assertRaises(TypeError) as raised: - basic_auth_protocol_factory(realm="auth-tests", credentials=42) - self.assertEqual(str(raised.exception), "invalid credentials argument: 42") - - create_protocol_multiple_credentials = basic_auth_protocol_factory( - realm="auth-tests", - credentials=[("hello", "iloveyou"), ("goodbye", "stillloveu")], - ) - - @with_server(create_protocol=create_protocol_multiple_credentials) - @with_client(user_info=("hello", "iloveyou")) - def test_basic_auth_server_multiple_credentials(self): - self.loop.run_until_complete(self.client.send("Hello!")) - self.loop.run_until_complete(self.client.recv()) - - def test_basic_auth_bad_multiple_credentials(self): - with self.assertRaises(TypeError) as raised: - basic_auth_protocol_factory( - realm="auth-tests", credentials=[("hello", "iloveyou"), 42] - ) - self.assertEqual( - str(raised.exception), - "invalid credentials argument: [('hello', 'iloveyou'), 42]", - ) - - async def check_credentials(username, password): - return password == "iloveyou" - - create_protocol_check_credentials = basic_auth_protocol_factory( - realm="auth-tests", - check_credentials=check_credentials, - ) - - @with_server(create_protocol=create_protocol_check_credentials) - @with_client(user_info=("hello", "iloveyou")) - def test_basic_auth_check_credentials(self): - self.loop.run_until_complete(self.client.send("Hello!")) - self.loop.run_until_complete(self.client.recv()) - - create_protocol_custom_protocol = basic_auth_protocol_factory( - realm="auth-tests", - credentials=[("hello", "iloveyou")], - create_protocol=CustomWebSocketServerProtocol, - ) - - @with_server(create_protocol=create_protocol_custom_protocol) - @with_client(user_info=("hello", "iloveyou")) - def test_basic_auth_custom_protocol(self): - self.assertTrue(CustomWebSocketServerProtocol.used) - del CustomWebSocketServerProtocol.used - self.loop.run_until_complete(self.client.send("Hello!")) - self.loop.run_until_complete(self.client.recv()) - - @with_server(create_protocol=create_protocol) - def test_basic_auth_missing_credentials(self): - with self.assertRaises(InvalidStatusCode) as raised: - self.start_client() - self.assertEqual(raised.exception.status_code, 401) - - @with_server(create_protocol=create_protocol) - def test_basic_auth_missing_credentials_details(self): - with self.assertRaises(urllib.error.HTTPError) as raised: - self.loop.run_until_complete(self.make_http_request()) - self.assertEqual(raised.exception.code, 401) - self.assertEqual( - raised.exception.headers["WWW-Authenticate"], - 'Basic realm="auth-tests", charset="UTF-8"', - ) - self.assertEqual(raised.exception.read().decode(), "Missing credentials\n") - - @with_server(create_protocol=create_protocol) - def test_basic_auth_unsupported_credentials(self): - with self.assertRaises(InvalidStatusCode) as raised: - self.start_client(extra_headers={"Authorization": "Digest ..."}) - self.assertEqual(raised.exception.status_code, 401) - - @with_server(create_protocol=create_protocol) - def test_basic_auth_unsupported_credentials_details(self): - with self.assertRaises(urllib.error.HTTPError) as raised: - self.loop.run_until_complete( - self.make_http_request(headers={"Authorization": "Digest ..."}) - ) - self.assertEqual(raised.exception.code, 401) - self.assertEqual( - raised.exception.headers["WWW-Authenticate"], - 'Basic realm="auth-tests", charset="UTF-8"', - ) - self.assertEqual(raised.exception.read().decode(), "Unsupported credentials\n") - - @with_server(create_protocol=create_protocol) - def test_basic_auth_invalid_credentials(self): - with self.assertRaises(InvalidStatusCode) as raised: - self.start_client(user_info=("hello", "ihateyou")) - self.assertEqual(raised.exception.status_code, 401) - - @with_server(create_protocol=create_protocol) - def test_basic_auth_invalid_credentials_details(self): - with self.assertRaises(urllib.error.HTTPError) as raised: - authorization = build_authorization_basic("hello", "ihateyou") - self.loop.run_until_complete( - self.make_http_request(headers={"Authorization": authorization}) - ) - self.assertEqual(raised.exception.code, 401) - self.assertEqual( - raised.exception.headers["WWW-Authenticate"], - 'Basic realm="auth-tests", charset="UTF-8"', - ) - self.assertEqual(raised.exception.read().decode(), "Invalid credentials\n") +# Check that the legacy auth module imports without an exception. +from websockets.auth import * # noqa diff --git a/tests/test_exports.py b/tests/test_exports.py index 7fcbc80e3..8e4330304 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -4,10 +4,12 @@ combined_exports = ( - websockets.auth.__all__ + websockets.legacy.auth.__all__ + + websockets.legacy.client.__all__ + + websockets.legacy.protocol.__all__ + + websockets.legacy.server.__all__ + websockets.client.__all__ + websockets.exceptions.__all__ - + websockets.protocol.__all__ + websockets.server.__all__ + websockets.typing.__all__ + websockets.uri.__all__ diff --git a/tests/test_framing.py b/tests/test_framing.py index 231cbf718..d6fa6352a 100644 --- a/tests/test_framing.py +++ b/tests/test_framing.py @@ -1,171 +1,9 @@ -import asyncio -import codecs -import unittest -import unittest.mock import warnings -from websockets.exceptions import PayloadTooBig, ProtocolError -from websockets.frames import OP_BINARY, OP_CLOSE, OP_PING, OP_PONG, OP_TEXT -from websockets.framing import * -from .utils import AsyncioTestCase - - -class FramingTests(AsyncioTestCase): - def decode(self, message, mask=False, max_size=None, extensions=None): - stream = asyncio.StreamReader(loop=self.loop) - stream.feed_data(message) - stream.feed_eof() - with warnings.catch_warnings(record=True): - frame = self.loop.run_until_complete( - Frame.read( - stream.readexactly, - mask=mask, - max_size=max_size, - extensions=extensions, - ) - ) - # Make sure all the data was consumed. - self.assertTrue(stream.at_eof()) - return frame - - def encode(self, frame, mask=False, extensions=None): - write = unittest.mock.Mock() - with warnings.catch_warnings(record=True): - frame.write(write, mask=mask, extensions=extensions) - # Ensure the entire frame is sent with a single call to write(). - # Multiple calls cause TCP fragmentation and degrade performance. - self.assertEqual(write.call_count, 1) - # The frame data is the single positional argument of that call. - self.assertEqual(len(write.call_args[0]), 1) - self.assertEqual(len(write.call_args[1]), 0) - return write.call_args[0][0] - - def round_trip(self, message, expected, mask=False, extensions=None): - decoded = self.decode(message, mask, extensions=extensions) - self.assertEqual(decoded, expected) - encoded = self.encode(decoded, mask, extensions=extensions) - if mask: # non-deterministic encoding - decoded = self.decode(encoded, mask, extensions=extensions) - self.assertEqual(decoded, expected) - else: # deterministic encoding - self.assertEqual(encoded, message) - - def test_text(self): - self.round_trip(b"\x81\x04Spam", Frame(True, OP_TEXT, b"Spam")) - - def test_text_masked(self): - self.round_trip( - b"\x81\x84\x5b\xfb\xe1\xa8\x08\x8b\x80\xc5", - Frame(True, OP_TEXT, b"Spam"), - mask=True, - ) - - def test_binary(self): - self.round_trip(b"\x82\x04Eggs", Frame(True, OP_BINARY, b"Eggs")) - - def test_binary_masked(self): - self.round_trip( - b"\x82\x84\x53\xcd\xe2\x89\x16\xaa\x85\xfa", - Frame(True, OP_BINARY, b"Eggs"), - mask=True, - ) - - def test_non_ascii_text(self): - self.round_trip( - b"\x81\x05caf\xc3\xa9", Frame(True, OP_TEXT, "café".encode("utf-8")) - ) - - def test_non_ascii_text_masked(self): - self.round_trip( - b"\x81\x85\x64\xbe\xee\x7e\x07\xdf\x88\xbd\xcd", - Frame(True, OP_TEXT, "café".encode("utf-8")), - mask=True, - ) - - def test_close(self): - self.round_trip(b"\x88\x00", Frame(True, OP_CLOSE, b"")) - - def test_ping(self): - self.round_trip(b"\x89\x04ping", Frame(True, OP_PING, b"ping")) - - def test_pong(self): - self.round_trip(b"\x8a\x04pong", Frame(True, OP_PONG, b"pong")) - - def test_long(self): - self.round_trip( - b"\x82\x7e\x00\x7e" + 126 * b"a", Frame(True, OP_BINARY, 126 * b"a") - ) - - def test_very_long(self): - self.round_trip( - b"\x82\x7f\x00\x00\x00\x00\x00\x01\x00\x00" + 65536 * b"a", - Frame(True, OP_BINARY, 65536 * b"a"), - ) - - def test_payload_too_big(self): - with self.assertRaises(PayloadTooBig): - self.decode(b"\x82\x7e\x04\x01" + 1025 * b"a", max_size=1024) - - def test_bad_reserved_bits(self): - for encoded in [b"\xc0\x00", b"\xa0\x00", b"\x90\x00"]: - with self.subTest(encoded=encoded): - with self.assertRaises(ProtocolError): - self.decode(encoded) - - def test_good_opcode(self): - for opcode in list(range(0x00, 0x03)) + list(range(0x08, 0x0B)): - encoded = bytes([0x80 | opcode, 0]) - with self.subTest(encoded=encoded): - self.decode(encoded) # does not raise an exception - - def test_bad_opcode(self): - for opcode in list(range(0x03, 0x08)) + list(range(0x0B, 0x10)): - encoded = bytes([0x80 | opcode, 0]) - with self.subTest(encoded=encoded): - with self.assertRaises(ProtocolError): - self.decode(encoded) - - def test_mask_flag(self): - # Mask flag correctly set. - self.decode(b"\x80\x80\x00\x00\x00\x00", mask=True) - # Mask flag incorrectly unset. - with self.assertRaises(ProtocolError): - self.decode(b"\x80\x80\x00\x00\x00\x00") - # Mask flag correctly unset. - self.decode(b"\x80\x00") - # Mask flag incorrectly set. - with self.assertRaises(ProtocolError): - self.decode(b"\x80\x00", mask=True) - - def test_control_frame_max_length(self): - # At maximum allowed length. - self.decode(b"\x88\x7e\x00\x7d" + 125 * b"a") - # Above maximum allowed length. - with self.assertRaises(ProtocolError): - self.decode(b"\x88\x7e\x00\x7e" + 126 * b"a") - - def test_fragmented_control_frame(self): - # Fin bit correctly set. - self.decode(b"\x88\x00") - # Fin bit incorrectly unset. - with self.assertRaises(ProtocolError): - self.decode(b"\x08\x00") - - def test_extensions(self): - class Rot13: - @staticmethod - def encode(frame): - assert frame.opcode == OP_TEXT - text = frame.data.decode() - data = codecs.encode(text, "rot13").encode() - return frame._replace(data=data) - - # This extensions is symmetrical. - @staticmethod - def decode(frame, *, max_size=None): - return Rot13.encode(frame) - - self.round_trip( - b"\x81\x05uryyb", Frame(True, OP_TEXT, b"hello"), extensions=[Rot13()] - ) +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", "websockets.framing is deprecated", DeprecationWarning + ) + # Check that the legacy framing module imports without an exception. + from websockets.framing import * # noqa diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 432c31ef5..f896fcae4 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -1,1489 +1,2 @@ -import asyncio -import contextlib -import sys -import unittest -import unittest.mock -import warnings - -from websockets.exceptions import ConnectionClosed, InvalidState -from websockets.frames import ( - OP_BINARY, - OP_CLOSE, - OP_CONT, - OP_PING, - OP_PONG, - OP_TEXT, - serialize_close, -) -from websockets.framing import Frame -from websockets.protocol import State, WebSocketCommonProtocol - -from .utils import MS, AsyncioTestCase - - -async def async_iterable(iterable): - for item in iterable: - yield item - - -class TransportMock(unittest.mock.Mock): - """ - Transport mock to control the protocol's inputs and outputs in tests. - - It calls the protocol's connection_made and connection_lost methods like - actual transports. - - It also calls the protocol's connection_open method to bypass the - WebSocket handshake. - - To simulate incoming data, tests call the protocol's data_received and - eof_received methods directly. - - They could also pause_writing and resume_writing to test flow control. - - """ - - # This should happen in __init__ but overriding Mock.__init__ is hard. - def setup_mock(self, loop, protocol): - self.loop = loop - self.protocol = protocol - self._eof = False - self._closing = False - # Simulate a successful TCP handshake. - self.protocol.connection_made(self) - # Simulate a successful WebSocket handshake. - self.protocol.connection_open() - - def can_write_eof(self): - return True - - def write_eof(self): - # When the protocol half-closes the TCP connection, it expects the - # other end to close it. Simulate that. - if not self._eof: - self.loop.call_soon(self.close) - self._eof = True - - def close(self): - # Simulate how actual transports drop the connection. - if not self._closing: - self.loop.call_soon(self.protocol.connection_lost, None) - self._closing = True - - def abort(self): - # Change this to an `if` if tests call abort() multiple times. - assert self.protocol.state is not State.CLOSED - self.loop.call_soon(self.protocol.connection_lost, None) - - -class CommonTests: - """ - Mixin that defines most tests but doesn't inherit unittest.TestCase. - - Tests are run by the ServerTests and ClientTests subclasses. - - """ - - def setUp(self): - super().setUp() - # Disable pings to make it easier to test what frames are sent exactly. - self.protocol = WebSocketCommonProtocol(ping_interval=None) - self.transport = TransportMock() - self.transport.setup_mock(self.loop, self.protocol) - - def tearDown(self): - self.transport.close() - self.loop.run_until_complete(self.protocol.close()) - super().tearDown() - - # Utilities for writing tests. - - def make_drain_slow(self, delay=MS): - # Process connection_made in order to initialize self.protocol.transport. - self.run_loop_once() - - original_drain = self.protocol._drain - - async def delayed_drain(): - await asyncio.sleep( - delay, loop=self.loop if sys.version_info[:2] < (3, 8) else None - ) - await original_drain() - - self.protocol._drain = delayed_drain - - close_frame = Frame(True, OP_CLOSE, serialize_close(1000, "close")) - local_close = Frame(True, OP_CLOSE, serialize_close(1000, "local")) - remote_close = Frame(True, OP_CLOSE, serialize_close(1000, "remote")) - - def receive_frame(self, frame): - """ - Make the protocol receive a frame. - - """ - write = self.protocol.data_received - mask = not self.protocol.is_client - frame.write(write, mask=mask) - - def receive_eof(self): - """ - Make the protocol receive the end of the data stream. - - Since ``WebSocketCommonProtocol.eof_received`` returns ``None``, an - actual transport would close itself after calling it. This function - emulates that behavior. - - """ - self.protocol.eof_received() - self.loop.call_soon(self.transport.close) - - def receive_eof_if_client(self): - """ - Like receive_eof, but only if this is the client side. - - Since the server is supposed to initiate the termination of the TCP - connection, this method helps making tests work for both sides. - - """ - if self.protocol.is_client: - self.receive_eof() - - def close_connection(self, code=1000, reason="close"): - """ - Execute a closing handshake. - - This puts the connection in the CLOSED state. - - """ - close_frame_data = serialize_close(code, reason) - # Prepare the response to the closing handshake from the remote side. - self.receive_frame(Frame(True, OP_CLOSE, close_frame_data)) - self.receive_eof_if_client() - # Trigger the closing handshake from the local side and complete it. - self.loop.run_until_complete(self.protocol.close(code, reason)) - # Empty the outgoing data stream so we can make assertions later on. - self.assertOneFrameSent(True, OP_CLOSE, close_frame_data) - - assert self.protocol.state is State.CLOSED - - def half_close_connection_local(self, code=1000, reason="close"): - """ - Start a closing handshake but do not complete it. - - The main difference with `close_connection` is that the connection is - left in the CLOSING state until the event loop runs again. - - The current implementation returns a task that must be awaited or - canceled, else asyncio complains about destroying a pending task. - - """ - close_frame_data = serialize_close(code, reason) - # Trigger the closing handshake from the local endpoint. - close_task = self.loop.create_task(self.protocol.close(code, reason)) - self.run_loop_once() # wait_for executes - self.run_loop_once() # write_frame executes - # Empty the outgoing data stream so we can make assertions later on. - self.assertOneFrameSent(True, OP_CLOSE, close_frame_data) - - assert self.protocol.state is State.CLOSING - - # Complete the closing sequence at 1ms intervals so the test can run - # at each point even it goes back to the event loop several times. - self.loop.call_later( - MS, self.receive_frame, Frame(True, OP_CLOSE, close_frame_data) - ) - self.loop.call_later(2 * MS, self.receive_eof_if_client) - - # This task must be awaited or canceled by the caller. - return close_task - - def half_close_connection_remote(self, code=1000, reason="close"): - """ - Receive a closing handshake but do not complete it. - - The main difference with `close_connection` is that the connection is - left in the CLOSING state until the event loop runs again. - - """ - # On the server side, websockets completes the closing handshake and - # closes the TCP connection immediately. Yield to the event loop after - # sending the close frame to run the test while the connection is in - # the CLOSING state. - if not self.protocol.is_client: - self.make_drain_slow() - - close_frame_data = serialize_close(code, reason) - # Trigger the closing handshake from the remote endpoint. - self.receive_frame(Frame(True, OP_CLOSE, close_frame_data)) - self.run_loop_once() # read_frame executes - # Empty the outgoing data stream so we can make assertions later on. - self.assertOneFrameSent(True, OP_CLOSE, close_frame_data) - - assert self.protocol.state is State.CLOSING - - # Complete the closing sequence at 1ms intervals so the test can run - # at each point even it goes back to the event loop several times. - self.loop.call_later(2 * MS, self.receive_eof_if_client) - - def process_invalid_frames(self): - """ - Make the protocol fail quickly after simulating invalid data. - - To achieve this, this function triggers the protocol's eof_received, - which interrupts pending reads waiting for more data. - - """ - self.run_loop_once() - self.receive_eof() - self.loop.run_until_complete(self.protocol.close_connection_task) - - def sent_frames(self): - """ - Read all frames sent to the transport. - - """ - stream = asyncio.StreamReader(loop=self.loop) - - for (data,), kw in self.transport.write.call_args_list: - stream.feed_data(data) - self.transport.write.call_args_list = [] - stream.feed_eof() - - frames = [] - while not stream.at_eof(): - frames.append( - self.loop.run_until_complete( - Frame.read(stream.readexactly, mask=self.protocol.is_client) - ) - ) - return frames - - def last_sent_frame(self): - """ - Read the last frame sent to the transport. - - This method assumes that at most one frame was sent. It raises an - AssertionError otherwise. - - """ - frames = self.sent_frames() - if frames: - assert len(frames) == 1 - return frames[0] - - def assertFramesSent(self, *frames): - self.assertEqual(self.sent_frames(), [Frame(*args) for args in frames]) - - def assertOneFrameSent(self, *args): - self.assertEqual(self.last_sent_frame(), Frame(*args)) - - def assertNoFrameSent(self): - self.assertIsNone(self.last_sent_frame()) - - def assertConnectionClosed(self, code, message): - # The following line guarantees that connection_lost was called. - self.assertEqual(self.protocol.state, State.CLOSED) - # A close frame was received. - self.assertEqual(self.protocol.close_code, code) - self.assertEqual(self.protocol.close_reason, message) - - def assertConnectionFailed(self, code, message): - # The following line guarantees that connection_lost was called. - self.assertEqual(self.protocol.state, State.CLOSED) - # No close frame was received. - self.assertEqual(self.protocol.close_code, 1006) - self.assertEqual(self.protocol.close_reason, "") - # A close frame was sent -- unless the connection was already lost. - if code == 1006: - self.assertNoFrameSent() - else: - self.assertOneFrameSent(True, OP_CLOSE, serialize_close(code, message)) - - @contextlib.contextmanager - def assertCompletesWithin(self, min_time, max_time): - t0 = self.loop.time() - yield - t1 = self.loop.time() - dt = t1 - t0 - self.assertGreaterEqual(dt, min_time, f"Too fast: {dt} < {min_time}") - self.assertLess(dt, max_time, f"Too slow: {dt} >= {max_time}") - - # Test constructor. - - def test_timeout_backwards_compatibility(self): - with warnings.catch_warnings(record=True) as recorded_warnings: - protocol = WebSocketCommonProtocol(timeout=5) - - self.assertEqual(protocol.close_timeout, 5) - - self.assertEqual(len(recorded_warnings), 1) - warning = recorded_warnings[0].message - self.assertEqual(str(warning), "rename timeout to close_timeout") - self.assertEqual(type(warning), DeprecationWarning) - - # Test public attributes. - - def test_local_address(self): - get_extra_info = unittest.mock.Mock(return_value=("host", 4312)) - self.transport.get_extra_info = get_extra_info - - self.assertEqual(self.protocol.local_address, ("host", 4312)) - get_extra_info.assert_called_with("sockname") - - def test_local_address_before_connection(self): - # Emulate the situation before connection_open() runs. - _transport = self.protocol.transport - del self.protocol.transport - try: - self.assertEqual(self.protocol.local_address, None) - finally: - self.protocol.transport = _transport - - def test_remote_address(self): - get_extra_info = unittest.mock.Mock(return_value=("host", 4312)) - self.transport.get_extra_info = get_extra_info - - self.assertEqual(self.protocol.remote_address, ("host", 4312)) - get_extra_info.assert_called_with("peername") - - def test_remote_address_before_connection(self): - # Emulate the situation before connection_open() runs. - _transport = self.protocol.transport - del self.protocol.transport - try: - self.assertEqual(self.protocol.remote_address, None) - finally: - self.protocol.transport = _transport - - def test_open(self): - self.assertTrue(self.protocol.open) - self.close_connection() - self.assertFalse(self.protocol.open) - - def test_closed(self): - self.assertFalse(self.protocol.closed) - self.close_connection() - self.assertTrue(self.protocol.closed) - - def test_wait_closed(self): - wait_closed = self.loop.create_task(self.protocol.wait_closed()) - self.assertFalse(wait_closed.done()) - self.close_connection() - self.assertTrue(wait_closed.done()) - - # Test the recv coroutine. - - def test_recv_text(self): - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, "café") - - def test_recv_binary(self): - self.receive_frame(Frame(True, OP_BINARY, b"tea")) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, b"tea") - - def test_recv_on_closing_connection_local(self): - close_task = self.half_close_connection_local() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.recv()) - - self.loop.run_until_complete(close_task) # cleanup - - def test_recv_on_closing_connection_remote(self): - self.half_close_connection_remote() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.recv()) - - def test_recv_on_closed_connection(self): - self.close_connection() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.recv()) - - def test_recv_protocol_error(self): - self.receive_frame(Frame(True, OP_CONT, "café".encode("utf-8"))) - self.process_invalid_frames() - self.assertConnectionFailed(1002, "") - - def test_recv_unicode_error(self): - self.receive_frame(Frame(True, OP_TEXT, "café".encode("latin-1"))) - self.process_invalid_frames() - self.assertConnectionFailed(1007, "") - - def test_recv_text_payload_too_big(self): - self.protocol.max_size = 1024 - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8") * 205)) - self.process_invalid_frames() - self.assertConnectionFailed(1009, "") - - def test_recv_binary_payload_too_big(self): - self.protocol.max_size = 1024 - self.receive_frame(Frame(True, OP_BINARY, b"tea" * 342)) - self.process_invalid_frames() - self.assertConnectionFailed(1009, "") - - def test_recv_text_no_max_size(self): - self.protocol.max_size = None # for test coverage - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8") * 205)) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, "café" * 205) - - def test_recv_binary_no_max_size(self): - self.protocol.max_size = None # for test coverage - self.receive_frame(Frame(True, OP_BINARY, b"tea" * 342)) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, b"tea" * 342) - - def test_recv_queue_empty(self): - recv = self.loop.create_task(self.protocol.recv()) - with self.assertRaises(asyncio.TimeoutError): - self.loop.run_until_complete( - asyncio.wait_for(asyncio.shield(recv), timeout=MS) - ) - - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) - data = self.loop.run_until_complete(recv) - self.assertEqual(data, "café") - - def test_recv_queue_full(self): - self.protocol.max_queue = 2 - # Test internals because it's hard to verify buffers from the outside. - self.assertEqual(list(self.protocol.messages), []) - - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) - self.run_loop_once() - self.assertEqual(list(self.protocol.messages), ["café"]) - - self.receive_frame(Frame(True, OP_BINARY, b"tea")) - self.run_loop_once() - self.assertEqual(list(self.protocol.messages), ["café", b"tea"]) - - self.receive_frame(Frame(True, OP_BINARY, b"milk")) - self.run_loop_once() - self.assertEqual(list(self.protocol.messages), ["café", b"tea"]) - - self.loop.run_until_complete(self.protocol.recv()) - self.run_loop_once() - self.assertEqual(list(self.protocol.messages), [b"tea", b"milk"]) - - self.loop.run_until_complete(self.protocol.recv()) - self.run_loop_once() - self.assertEqual(list(self.protocol.messages), [b"milk"]) - - self.loop.run_until_complete(self.protocol.recv()) - self.run_loop_once() - self.assertEqual(list(self.protocol.messages), []) - - def test_recv_queue_no_limit(self): - self.protocol.max_queue = None - - for _ in range(100): - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) - self.run_loop_once() - - # Incoming message queue can contain at least 100 messages. - self.assertEqual(list(self.protocol.messages), ["café"] * 100) - - for _ in range(100): - self.loop.run_until_complete(self.protocol.recv()) - - self.assertEqual(list(self.protocol.messages), []) - - def test_recv_other_error(self): - async def read_message(): - raise Exception("BOOM") - - self.protocol.read_message = read_message - self.process_invalid_frames() - self.assertConnectionFailed(1011, "") - - def test_recv_canceled(self): - recv = self.loop.create_task(self.protocol.recv()) - self.loop.call_soon(recv.cancel) - - with self.assertRaises(asyncio.CancelledError): - self.loop.run_until_complete(recv) - - # The next frame doesn't disappear in a vacuum (it used to). - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, "café") - - def test_recv_canceled_race_condition(self): - recv = self.loop.create_task( - asyncio.wait_for(self.protocol.recv(), timeout=0.000_001) - ) - self.loop.call_soon( - self.receive_frame, Frame(True, OP_TEXT, "café".encode("utf-8")) - ) - - with self.assertRaises(asyncio.TimeoutError): - self.loop.run_until_complete(recv) - - # The previous frame doesn't disappear in a vacuum (it used to). - self.receive_frame(Frame(True, OP_TEXT, "tea".encode("utf-8"))) - data = self.loop.run_until_complete(self.protocol.recv()) - # If we're getting "tea" there, it means "café" was swallowed (ha, ha). - self.assertEqual(data, "café") - - def test_recv_when_transfer_data_cancelled(self): - # Clog incoming queue. - self.protocol.max_queue = 1 - self.receive_frame(Frame(True, OP_TEXT, "café".encode("utf-8"))) - self.receive_frame(Frame(True, OP_BINARY, b"tea")) - self.run_loop_once() - - # Flow control kicks in (check with an implementation detail). - self.assertFalse(self.protocol._put_message_waiter.done()) - - # Schedule recv(). - recv = self.loop.create_task(self.protocol.recv()) - - # Cancel transfer_data_task (again, implementation detail). - self.protocol.fail_connection() - self.run_loop_once() - self.assertTrue(self.protocol.transfer_data_task.cancelled()) - - # recv() completes properly. - self.assertEqual(self.loop.run_until_complete(recv), "café") - - def test_recv_prevents_concurrent_calls(self): - recv = self.loop.create_task(self.protocol.recv()) - - with self.assertRaises(RuntimeError) as raised: - self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual( - str(raised.exception), - "cannot call recv while another coroutine " - "is already waiting for the next message", - ) - recv.cancel() - - # Test the send coroutine. - - def test_send_text(self): - self.loop.run_until_complete(self.protocol.send("café")) - self.assertOneFrameSent(True, OP_TEXT, "café".encode("utf-8")) - - def test_send_binary(self): - self.loop.run_until_complete(self.protocol.send(b"tea")) - self.assertOneFrameSent(True, OP_BINARY, b"tea") - - def test_send_binary_from_bytearray(self): - self.loop.run_until_complete(self.protocol.send(bytearray(b"tea"))) - self.assertOneFrameSent(True, OP_BINARY, b"tea") - - def test_send_binary_from_memoryview(self): - self.loop.run_until_complete(self.protocol.send(memoryview(b"tea"))) - self.assertOneFrameSent(True, OP_BINARY, b"tea") - - def test_send_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete(self.protocol.send(memoryview(b"tteeaa")[::2])) - self.assertOneFrameSent(True, OP_BINARY, b"tea") - - def test_send_dict(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete(self.protocol.send({"not": "encoded"})) - self.assertNoFrameSent() - - def test_send_type_error(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete(self.protocol.send(42)) - self.assertNoFrameSent() - - def test_send_iterable_text(self): - self.loop.run_until_complete(self.protocol.send(["ca", "fé"])) - self.assertFramesSent( - (False, OP_TEXT, "ca".encode("utf-8")), - (False, OP_CONT, "fé".encode("utf-8")), - (True, OP_CONT, "".encode("utf-8")), - ) - - def test_send_iterable_binary(self): - self.loop.run_until_complete(self.protocol.send([b"te", b"a"])) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_iterable_binary_from_bytearray(self): - self.loop.run_until_complete( - self.protocol.send([bytearray(b"te"), bytearray(b"a")]) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_iterable_binary_from_memoryview(self): - self.loop.run_until_complete( - self.protocol.send([memoryview(b"te"), memoryview(b"a")]) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_iterable_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete( - self.protocol.send([memoryview(b"ttee")[::2], memoryview(b"aa")[::2]]) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_empty_iterable(self): - self.loop.run_until_complete(self.protocol.send([])) - self.assertNoFrameSent() - - def test_send_iterable_type_error(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete(self.protocol.send([42])) - self.assertNoFrameSent() - - def test_send_iterable_mixed_type_error(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete(self.protocol.send(["café", b"tea"])) - self.assertFramesSent( - (False, OP_TEXT, "café".encode("utf-8")), - (True, OP_CLOSE, serialize_close(1011, "")), - ) - - def test_send_iterable_prevents_concurrent_send(self): - self.make_drain_slow(2 * MS) - - async def send_iterable(): - await self.protocol.send(["ca", "fé"]) - - async def send_concurrent(): - await asyncio.sleep(MS) - await self.protocol.send(b"tea") - - self.loop.run_until_complete(asyncio.gather(send_iterable(), send_concurrent())) - self.assertFramesSent( - (False, OP_TEXT, "ca".encode("utf-8")), - (False, OP_CONT, "fé".encode("utf-8")), - (True, OP_CONT, "".encode("utf-8")), - (True, OP_BINARY, b"tea"), - ) - - def test_send_async_iterable_text(self): - self.loop.run_until_complete(self.protocol.send(async_iterable(["ca", "fé"]))) - self.assertFramesSent( - (False, OP_TEXT, "ca".encode("utf-8")), - (False, OP_CONT, "fé".encode("utf-8")), - (True, OP_CONT, "".encode("utf-8")), - ) - - def test_send_async_iterable_binary(self): - self.loop.run_until_complete(self.protocol.send(async_iterable([b"te", b"a"]))) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_async_iterable_binary_from_bytearray(self): - self.loop.run_until_complete( - self.protocol.send(async_iterable([bytearray(b"te"), bytearray(b"a")])) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_async_iterable_binary_from_memoryview(self): - self.loop.run_until_complete( - self.protocol.send(async_iterable([memoryview(b"te"), memoryview(b"a")])) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_async_iterable_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete( - self.protocol.send( - async_iterable([memoryview(b"ttee")[::2], memoryview(b"aa")[::2]]) - ) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - - def test_send_empty_async_iterable(self): - self.loop.run_until_complete(self.protocol.send(async_iterable([]))) - self.assertNoFrameSent() - - def test_send_async_iterable_type_error(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete(self.protocol.send(async_iterable([42]))) - self.assertNoFrameSent() - - def test_send_async_iterable_mixed_type_error(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete( - self.protocol.send(async_iterable(["café", b"tea"])) - ) - self.assertFramesSent( - (False, OP_TEXT, "café".encode("utf-8")), - (True, OP_CLOSE, serialize_close(1011, "")), - ) - - def test_send_async_iterable_prevents_concurrent_send(self): - self.make_drain_slow(2 * MS) - - async def send_async_iterable(): - await self.protocol.send(async_iterable(["ca", "fé"])) - - async def send_concurrent(): - await asyncio.sleep(MS) - await self.protocol.send(b"tea") - - self.loop.run_until_complete( - asyncio.gather(send_async_iterable(), send_concurrent()) - ) - self.assertFramesSent( - (False, OP_TEXT, "ca".encode("utf-8")), - (False, OP_CONT, "fé".encode("utf-8")), - (True, OP_CONT, "".encode("utf-8")), - (True, OP_BINARY, b"tea"), - ) - - def test_send_on_closing_connection_local(self): - close_task = self.half_close_connection_local() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.send("foobar")) - - self.assertNoFrameSent() - - self.loop.run_until_complete(close_task) # cleanup - - def test_send_on_closing_connection_remote(self): - self.half_close_connection_remote() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.send("foobar")) - - self.assertNoFrameSent() - - def test_send_on_closed_connection(self): - self.close_connection() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.send("foobar")) - - self.assertNoFrameSent() - - # Test the ping coroutine. - - def test_ping_default(self): - self.loop.run_until_complete(self.protocol.ping()) - # With our testing tools, it's more convenient to extract the expected - # ping data from the library's internals than from the frame sent. - ping_data = next(iter(self.protocol.pings)) - self.assertIsInstance(ping_data, bytes) - self.assertEqual(len(ping_data), 4) - self.assertOneFrameSent(True, OP_PING, ping_data) - - def test_ping_text(self): - self.loop.run_until_complete(self.protocol.ping("café")) - self.assertOneFrameSent(True, OP_PING, "café".encode("utf-8")) - - def test_ping_binary(self): - self.loop.run_until_complete(self.protocol.ping(b"tea")) - self.assertOneFrameSent(True, OP_PING, b"tea") - - def test_ping_binary_from_bytearray(self): - self.loop.run_until_complete(self.protocol.ping(bytearray(b"tea"))) - self.assertOneFrameSent(True, OP_PING, b"tea") - - def test_ping_binary_from_memoryview(self): - self.loop.run_until_complete(self.protocol.ping(memoryview(b"tea"))) - self.assertOneFrameSent(True, OP_PING, b"tea") - - def test_ping_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete(self.protocol.ping(memoryview(b"tteeaa")[::2])) - self.assertOneFrameSent(True, OP_PING, b"tea") - - def test_ping_type_error(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete(self.protocol.ping(42)) - self.assertNoFrameSent() - - def test_ping_on_closing_connection_local(self): - close_task = self.half_close_connection_local() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.ping()) - - self.assertNoFrameSent() - - self.loop.run_until_complete(close_task) # cleanup - - def test_ping_on_closing_connection_remote(self): - self.half_close_connection_remote() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.ping()) - - self.assertNoFrameSent() - - def test_ping_on_closed_connection(self): - self.close_connection() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.ping()) - - self.assertNoFrameSent() - - # Test the pong coroutine. - - def test_pong_default(self): - self.loop.run_until_complete(self.protocol.pong()) - self.assertOneFrameSent(True, OP_PONG, b"") - - def test_pong_text(self): - self.loop.run_until_complete(self.protocol.pong("café")) - self.assertOneFrameSent(True, OP_PONG, "café".encode("utf-8")) - - def test_pong_binary(self): - self.loop.run_until_complete(self.protocol.pong(b"tea")) - self.assertOneFrameSent(True, OP_PONG, b"tea") - - def test_pong_binary_from_bytearray(self): - self.loop.run_until_complete(self.protocol.pong(bytearray(b"tea"))) - self.assertOneFrameSent(True, OP_PONG, b"tea") - - def test_pong_binary_from_memoryview(self): - self.loop.run_until_complete(self.protocol.pong(memoryview(b"tea"))) - self.assertOneFrameSent(True, OP_PONG, b"tea") - - def test_pong_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete(self.protocol.pong(memoryview(b"tteeaa")[::2])) - self.assertOneFrameSent(True, OP_PONG, b"tea") - - def test_pong_type_error(self): - with self.assertRaises(TypeError): - self.loop.run_until_complete(self.protocol.pong(42)) - self.assertNoFrameSent() - - def test_pong_on_closing_connection_local(self): - close_task = self.half_close_connection_local() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.pong()) - - self.assertNoFrameSent() - - self.loop.run_until_complete(close_task) # cleanup - - def test_pong_on_closing_connection_remote(self): - self.half_close_connection_remote() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.pong()) - - self.assertNoFrameSent() - - def test_pong_on_closed_connection(self): - self.close_connection() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.pong()) - - self.assertNoFrameSent() - - # Test the protocol's logic for acknowledging pings with pongs. - - def test_answer_ping(self): - self.receive_frame(Frame(True, OP_PING, b"test")) - self.run_loop_once() - self.assertOneFrameSent(True, OP_PONG, b"test") - - def test_ignore_pong(self): - self.receive_frame(Frame(True, OP_PONG, b"test")) - self.run_loop_once() - self.assertNoFrameSent() - - def test_acknowledge_ping(self): - ping = self.loop.run_until_complete(self.protocol.ping()) - self.assertFalse(ping.done()) - ping_frame = self.last_sent_frame() - pong_frame = Frame(True, OP_PONG, ping_frame.data) - self.receive_frame(pong_frame) - self.run_loop_once() - self.run_loop_once() - self.assertTrue(ping.done()) - - def test_abort_ping(self): - ping = self.loop.run_until_complete(self.protocol.ping()) - # Remove the frame from the buffer, else close_connection() complains. - self.last_sent_frame() - self.assertFalse(ping.done()) - self.close_connection() - self.assertTrue(ping.done()) - self.assertIsInstance(ping.exception(), ConnectionClosed) - - def test_abort_ping_does_not_log_exception_if_not_retreived(self): - self.loop.run_until_complete(self.protocol.ping()) - # Get the internal Future, which isn't directly returned by ping(). - (ping,) = self.protocol.pings.values() - # Remove the frame from the buffer, else close_connection() complains. - self.last_sent_frame() - self.close_connection() - # Check a private attribute, for lack of a better solution. - self.assertFalse(ping._log_traceback) - - def test_acknowledge_previous_pings(self): - pings = [ - (self.loop.run_until_complete(self.protocol.ping()), self.last_sent_frame()) - for i in range(3) - ] - # Unsolicited pong doesn't acknowledge pings - self.receive_frame(Frame(True, OP_PONG, b"")) - self.run_loop_once() - self.run_loop_once() - self.assertFalse(pings[0][0].done()) - self.assertFalse(pings[1][0].done()) - self.assertFalse(pings[2][0].done()) - # Pong acknowledges all previous pings - self.receive_frame(Frame(True, OP_PONG, pings[1][1].data)) - self.run_loop_once() - self.run_loop_once() - self.assertTrue(pings[0][0].done()) - self.assertTrue(pings[1][0].done()) - self.assertFalse(pings[2][0].done()) - - def test_acknowledge_aborted_ping(self): - ping = self.loop.run_until_complete(self.protocol.ping()) - ping_frame = self.last_sent_frame() - # Clog incoming queue. This lets connection_lost() abort pending pings - # with a ConnectionClosed exception before transfer_data_task - # terminates and close_connection cancels keepalive_ping_task. - self.protocol.max_queue = 1 - self.receive_frame(Frame(True, OP_TEXT, b"1")) - self.receive_frame(Frame(True, OP_TEXT, b"2")) - # Add pong frame to the queue. - pong_frame = Frame(True, OP_PONG, ping_frame.data) - self.receive_frame(pong_frame) - # Connection drops. - self.receive_eof() - self.loop.run_until_complete(self.protocol.wait_closed()) - # Ping receives a ConnectionClosed exception. - with self.assertRaises(ConnectionClosed): - ping.result() - - # transfer_data doesn't crash, which would be logged. - with self.assertNoLogs(): - # Unclog incoming queue. - self.loop.run_until_complete(self.protocol.recv()) - self.loop.run_until_complete(self.protocol.recv()) - - def test_canceled_ping(self): - ping = self.loop.run_until_complete(self.protocol.ping()) - ping_frame = self.last_sent_frame() - ping.cancel() - pong_frame = Frame(True, OP_PONG, ping_frame.data) - self.receive_frame(pong_frame) - self.run_loop_once() - self.run_loop_once() - self.assertTrue(ping.cancelled()) - - def test_duplicate_ping(self): - self.loop.run_until_complete(self.protocol.ping(b"foobar")) - self.assertOneFrameSent(True, OP_PING, b"foobar") - with self.assertRaises(ValueError): - self.loop.run_until_complete(self.protocol.ping(b"foobar")) - self.assertNoFrameSent() - - # Test the protocol's logic for rebuilding fragmented messages. - - def test_fragmented_text(self): - self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) - self.receive_frame(Frame(True, OP_CONT, "fé".encode("utf-8"))) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, "café") - - def test_fragmented_binary(self): - self.receive_frame(Frame(False, OP_BINARY, b"t")) - self.receive_frame(Frame(False, OP_CONT, b"e")) - self.receive_frame(Frame(True, OP_CONT, b"a")) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, b"tea") - - def test_fragmented_text_payload_too_big(self): - self.protocol.max_size = 1024 - self.receive_frame(Frame(False, OP_TEXT, "café".encode("utf-8") * 100)) - self.receive_frame(Frame(True, OP_CONT, "café".encode("utf-8") * 105)) - self.process_invalid_frames() - self.assertConnectionFailed(1009, "") - - def test_fragmented_binary_payload_too_big(self): - self.protocol.max_size = 1024 - self.receive_frame(Frame(False, OP_BINARY, b"tea" * 171)) - self.receive_frame(Frame(True, OP_CONT, b"tea" * 171)) - self.process_invalid_frames() - self.assertConnectionFailed(1009, "") - - def test_fragmented_text_no_max_size(self): - self.protocol.max_size = None # for test coverage - self.receive_frame(Frame(False, OP_TEXT, "café".encode("utf-8") * 100)) - self.receive_frame(Frame(True, OP_CONT, "café".encode("utf-8") * 105)) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, "café" * 205) - - def test_fragmented_binary_no_max_size(self): - self.protocol.max_size = None # for test coverage - self.receive_frame(Frame(False, OP_BINARY, b"tea" * 171)) - self.receive_frame(Frame(True, OP_CONT, b"tea" * 171)) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, b"tea" * 342) - - def test_control_frame_within_fragmented_text(self): - self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) - self.receive_frame(Frame(True, OP_PING, b"")) - self.receive_frame(Frame(True, OP_CONT, "fé".encode("utf-8"))) - data = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(data, "café") - self.assertOneFrameSent(True, OP_PONG, b"") - - def test_unterminated_fragmented_text(self): - self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) - # Missing the second part of the fragmented frame. - self.receive_frame(Frame(True, OP_BINARY, b"tea")) - self.process_invalid_frames() - self.assertConnectionFailed(1002, "") - - def test_close_handshake_in_fragmented_text(self): - self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) - self.receive_frame(Frame(True, OP_CLOSE, b"")) - self.process_invalid_frames() - # The RFC may have overlooked this case: it says that control frames - # can be interjected in the middle of a fragmented message and that a - # close frame must be echoed. Even though there's an unterminated - # message, technically, the closing handshake was successful. - self.assertConnectionClosed(1005, "") - - def test_connection_close_in_fragmented_text(self): - self.receive_frame(Frame(False, OP_TEXT, "ca".encode("utf-8"))) - self.process_invalid_frames() - self.assertConnectionFailed(1006, "") - - # Test miscellaneous code paths to ensure full coverage. - - def test_connection_lost(self): - # Test calling connection_lost without going through close_connection. - self.protocol.connection_lost(None) - - self.assertConnectionFailed(1006, "") - - def test_ensure_open_before_opening_handshake(self): - # Simulate a bug by forcibly reverting the protocol state. - self.protocol.state = State.CONNECTING - - with self.assertRaises(InvalidState): - self.loop.run_until_complete(self.protocol.ensure_open()) - - def test_ensure_open_during_unclean_close(self): - # Process connection_made in order to start transfer_data_task. - self.run_loop_once() - - # Ensure the test terminates quickly. - self.loop.call_later(MS, self.receive_eof_if_client) - - # Simulate the case when close() times out sending a close frame. - self.protocol.fail_connection() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.ensure_open()) - - def test_legacy_recv(self): - # By default legacy_recv in disabled. - self.assertEqual(self.protocol.legacy_recv, False) - - self.close_connection() - - # Enable legacy_recv. - self.protocol.legacy_recv = True - - # Now recv() returns None instead of raising ConnectionClosed. - self.assertIsNone(self.loop.run_until_complete(self.protocol.recv())) - - def test_connection_closed_attributes(self): - self.close_connection() - - with self.assertRaises(ConnectionClosed) as context: - self.loop.run_until_complete(self.protocol.recv()) - - connection_closed_exc = context.exception - self.assertEqual(connection_closed_exc.code, 1000) - self.assertEqual(connection_closed_exc.reason, "close") - - # Test the protocol logic for sending keepalive pings. - - def restart_protocol_with_keepalive_ping( - self, ping_interval=3 * MS, ping_timeout=3 * MS - ): - initial_protocol = self.protocol - # copied from tearDown - self.transport.close() - self.loop.run_until_complete(self.protocol.close()) - # copied from setUp, but enables keepalive pings - self.protocol = WebSocketCommonProtocol( - ping_interval=ping_interval, ping_timeout=ping_timeout - ) - self.transport = TransportMock() - self.transport.setup_mock(self.loop, self.protocol) - self.protocol.is_client = initial_protocol.is_client - self.protocol.side = initial_protocol.side - - def test_keepalive_ping(self): - self.restart_protocol_with_keepalive_ping() - - # Ping is sent at 3ms and acknowledged at 4ms. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - (ping_1,) = tuple(self.protocol.pings) - self.assertOneFrameSent(True, OP_PING, ping_1) - self.receive_frame(Frame(True, OP_PONG, ping_1)) - - # Next ping is sent at 7ms. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - (ping_2,) = tuple(self.protocol.pings) - self.assertOneFrameSent(True, OP_PING, ping_2) - - # The keepalive ping task goes on. - self.assertFalse(self.protocol.keepalive_ping_task.done()) - - def test_keepalive_ping_not_acknowledged_closes_connection(self): - self.restart_protocol_with_keepalive_ping() - - # Ping is sent at 3ms and not acknowleged. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - (ping_1,) = tuple(self.protocol.pings) - self.assertOneFrameSent(True, OP_PING, ping_1) - - # Connection is closed at 6ms. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - self.assertOneFrameSent(True, OP_CLOSE, serialize_close(1011, "")) - - # The keepalive ping task is complete. - self.assertEqual(self.protocol.keepalive_ping_task.result(), None) - - def test_keepalive_ping_stops_when_connection_closing(self): - self.restart_protocol_with_keepalive_ping() - close_task = self.half_close_connection_local() - - # No ping sent at 3ms because the closing handshake is in progress. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - self.assertNoFrameSent() - - # The keepalive ping task terminated. - self.assertTrue(self.protocol.keepalive_ping_task.cancelled()) - - self.loop.run_until_complete(close_task) # cleanup - - def test_keepalive_ping_stops_when_connection_closed(self): - self.restart_protocol_with_keepalive_ping() - self.close_connection() - - # The keepalive ping task terminated. - self.assertTrue(self.protocol.keepalive_ping_task.cancelled()) - - def test_keepalive_ping_does_not_crash_when_connection_lost(self): - self.restart_protocol_with_keepalive_ping() - # Clog incoming queue. This lets connection_lost() abort pending pings - # with a ConnectionClosed exception before transfer_data_task - # terminates and close_connection cancels keepalive_ping_task. - self.protocol.max_queue = 1 - self.receive_frame(Frame(True, OP_TEXT, b"1")) - self.receive_frame(Frame(True, OP_TEXT, b"2")) - # Ping is sent at 3ms. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - (ping_waiter,) = tuple(self.protocol.pings.values()) - # Connection drops. - self.receive_eof() - self.loop.run_until_complete(self.protocol.wait_closed()) - - # The ping waiter receives a ConnectionClosed exception. - with self.assertRaises(ConnectionClosed): - ping_waiter.result() - # The keepalive ping task terminated properly. - self.assertIsNone(self.protocol.keepalive_ping_task.result()) - - # Unclog incoming queue to terminate the test quickly. - self.loop.run_until_complete(self.protocol.recv()) - self.loop.run_until_complete(self.protocol.recv()) - - def test_keepalive_ping_with_no_ping_interval(self): - self.restart_protocol_with_keepalive_ping(ping_interval=None) - - # No ping is sent at 3ms. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - self.assertNoFrameSent() - - def test_keepalive_ping_with_no_ping_timeout(self): - self.restart_protocol_with_keepalive_ping(ping_timeout=None) - - # Ping is sent at 3ms and not acknowleged. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - (ping_1,) = tuple(self.protocol.pings) - self.assertOneFrameSent(True, OP_PING, ping_1) - - # Next ping is sent at 7ms anyway. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - ping_1_again, ping_2 = tuple(self.protocol.pings) - self.assertEqual(ping_1, ping_1_again) - self.assertOneFrameSent(True, OP_PING, ping_2) - - # The keepalive ping task goes on. - self.assertFalse(self.protocol.keepalive_ping_task.done()) - - def test_keepalive_ping_unexpected_error(self): - self.restart_protocol_with_keepalive_ping() - - async def ping(): - raise Exception("BOOM") - - self.protocol.ping = ping - - # The keepalive ping task fails when sending a ping at 3ms. - self.loop.run_until_complete(asyncio.sleep(4 * MS)) - - # The keepalive ping task is complete. - # It logs and swallows the exception. - self.assertEqual(self.protocol.keepalive_ping_task.result(), None) - - # Test the protocol logic for closing the connection. - - def test_local_close(self): - # Emulate how the remote endpoint answers the closing handshake. - self.loop.call_later(MS, self.receive_frame, self.close_frame) - self.loop.call_later(MS, self.receive_eof_if_client) - - # Run the closing handshake. - self.loop.run_until_complete(self.protocol.close(reason="close")) - - self.assertConnectionClosed(1000, "close") - self.assertOneFrameSent(*self.close_frame) - - # Closing the connection again is a no-op. - self.loop.run_until_complete(self.protocol.close(reason="oh noes!")) - - self.assertConnectionClosed(1000, "close") - self.assertNoFrameSent() - - def test_remote_close(self): - # Emulate how the remote endpoint initiates the closing handshake. - self.loop.call_later(MS, self.receive_frame, self.close_frame) - self.loop.call_later(MS, self.receive_eof_if_client) - - # Wait for some data in order to process the handshake. - # After recv() raises ConnectionClosed, the connection is closed. - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(self.protocol.recv()) - - self.assertConnectionClosed(1000, "close") - self.assertOneFrameSent(*self.close_frame) - - # Closing the connection again is a no-op. - self.loop.run_until_complete(self.protocol.close(reason="oh noes!")) - - self.assertConnectionClosed(1000, "close") - self.assertNoFrameSent() - - def test_remote_close_and_connection_lost(self): - self.make_drain_slow() - # Drop the connection right after receiving a close frame, - # which prevents echoing the close frame properly. - self.receive_frame(self.close_frame) - self.receive_eof() - - with self.assertNoLogs(): - self.loop.run_until_complete(self.protocol.close(reason="oh noes!")) - - self.assertConnectionClosed(1000, "close") - self.assertOneFrameSent(*self.close_frame) - - def test_simultaneous_close(self): - # Receive the incoming close frame right after self.protocol.close() - # starts executing. This reproduces the error described in: - # https://github.com/aaugustin/websockets/issues/339 - self.loop.call_soon(self.receive_frame, self.remote_close) - self.loop.call_soon(self.receive_eof_if_client) - - self.loop.run_until_complete(self.protocol.close(reason="local")) - - self.assertConnectionClosed(1000, "remote") - # The current implementation sends a close frame in response to the - # close frame received from the remote end. It skips the close frame - # that should be sent as a result of calling close(). - self.assertOneFrameSent(*self.remote_close) - - def test_close_preserves_incoming_frames(self): - self.receive_frame(Frame(True, OP_TEXT, b"hello")) - - self.loop.call_later(MS, self.receive_frame, self.close_frame) - self.loop.call_later(MS, self.receive_eof_if_client) - self.loop.run_until_complete(self.protocol.close(reason="close")) - - self.assertConnectionClosed(1000, "close") - self.assertOneFrameSent(*self.close_frame) - - next_message = self.loop.run_until_complete(self.protocol.recv()) - self.assertEqual(next_message, "hello") - - def test_close_protocol_error(self): - invalid_close_frame = Frame(True, OP_CLOSE, b"\x00") - self.receive_frame(invalid_close_frame) - self.receive_eof_if_client() - self.run_loop_once() - self.loop.run_until_complete(self.protocol.close(reason="close")) - - self.assertConnectionFailed(1002, "") - - def test_close_connection_lost(self): - self.receive_eof() - self.run_loop_once() - self.loop.run_until_complete(self.protocol.close(reason="close")) - - self.assertConnectionFailed(1006, "") - - def test_local_close_during_recv(self): - recv = self.loop.create_task(self.protocol.recv()) - - self.loop.call_later(MS, self.receive_frame, self.close_frame) - self.loop.call_later(MS, self.receive_eof_if_client) - - self.loop.run_until_complete(self.protocol.close(reason="close")) - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(recv) - - self.assertConnectionClosed(1000, "close") - - # There is no test_remote_close_during_recv because it would be identical - # to test_remote_close. - - def test_remote_close_during_send(self): - self.make_drain_slow() - send = self.loop.create_task(self.protocol.send("hello")) - - self.receive_frame(self.close_frame) - self.receive_eof() - - with self.assertRaises(ConnectionClosed): - self.loop.run_until_complete(send) - - self.assertConnectionClosed(1000, "close") - - # There is no test_local_close_during_send because this cannot really - # happen, considering that writes are serialized. - - -class ServerTests(CommonTests, AsyncioTestCase): - def setUp(self): - super().setUp() - self.protocol.is_client = False - self.protocol.side = "server" - - def test_local_close_send_close_frame_timeout(self): - self.protocol.close_timeout = 10 * MS - self.make_drain_slow(50 * MS) - # If we can't send a close frame, time out in 10ms. - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(9 * MS, 19 * MS): - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1006, "") - - def test_local_close_receive_close_frame_timeout(self): - self.protocol.close_timeout = 10 * MS - # If the client doesn't send a close frame, time out in 10ms. - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(9 * MS, 19 * MS): - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1006, "") - - def test_local_close_connection_lost_timeout_after_write_eof(self): - self.protocol.close_timeout = 10 * MS - # If the client doesn't close its side of the TCP connection after we - # half-close our side with write_eof(), time out in 10ms. - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(9 * MS, 19 * MS): - # HACK: disable write_eof => other end drops connection emulation. - self.transport._eof = True - self.receive_frame(self.close_frame) - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1000, "close") - - def test_local_close_connection_lost_timeout_after_close(self): - self.protocol.close_timeout = 10 * MS - # If the client doesn't close its side of the TCP connection after we - # half-close our side with write_eof() and close it with close(), time - # out in 20ms. - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(19 * MS, 29 * MS): - # HACK: disable write_eof => other end drops connection emulation. - self.transport._eof = True - # HACK: disable close => other end drops connection emulation. - self.transport._closing = True - self.receive_frame(self.close_frame) - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1000, "close") - - -class ClientTests(CommonTests, AsyncioTestCase): - def setUp(self): - super().setUp() - self.protocol.is_client = True - self.protocol.side = "client" - - def test_local_close_send_close_frame_timeout(self): - self.protocol.close_timeout = 10 * MS - self.make_drain_slow(50 * MS) - # If we can't send a close frame, time out in 20ms. - # - 10ms waiting for sending a close frame - # - 10ms waiting for receiving a half-close - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(19 * MS, 29 * MS): - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1006, "") - - def test_local_close_receive_close_frame_timeout(self): - self.protocol.close_timeout = 10 * MS - # If the server doesn't send a close frame, time out in 20ms: - # - 10ms waiting for receiving a close frame - # - 10ms waiting for receiving a half-close - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(19 * MS, 29 * MS): - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1006, "") - - def test_local_close_connection_lost_timeout_after_write_eof(self): - self.protocol.close_timeout = 10 * MS - # If the server doesn't half-close its side of the TCP connection - # after we send a close frame, time out in 20ms: - # - 10ms waiting for receiving a half-close - # - 10ms waiting for receiving a close after write_eof - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(19 * MS, 29 * MS): - # HACK: disable write_eof => other end drops connection emulation. - self.transport._eof = True - self.receive_frame(self.close_frame) - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1000, "close") - - def test_local_close_connection_lost_timeout_after_close(self): - self.protocol.close_timeout = 10 * MS - # If the client doesn't close its side of the TCP connection after we - # half-close our side with write_eof() and close it with close(), time - # out in 20ms. - # - 10ms waiting for receiving a half-close - # - 10ms waiting for receiving a close after write_eof - # - 10ms waiting for receiving a close after close - # Check the timing within -1/+9ms for robustness. - with self.assertCompletesWithin(29 * MS, 39 * MS): - # HACK: disable write_eof => other end drops connection emulation. - self.transport._eof = True - # HACK: disable close => other end drops connection emulation. - self.transport._closing = True - self.receive_frame(self.close_frame) - self.loop.run_until_complete(self.protocol.close(reason="close")) - self.assertConnectionClosed(1000, "close") +# Check that the legacy protocol module imports without an exception. +from websockets.protocol import * # noqa diff --git a/tests/utils.py b/tests/utils.py index 790d25687..ac891a0fd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,10 +1,4 @@ -import asyncio -import contextlib import email.utils -import functools -import logging -import os -import time import unittest @@ -27,89 +21,3 @@ def assertGeneratorReturns(self, gen): with self.assertRaises(StopIteration) as raised: next(gen) return raised.exception.value - - -class AsyncioTestCase(unittest.TestCase): - """ - Base class for tests that sets up an isolated event loop for each test. - - """ - - def __init_subclass__(cls, **kwargs): - """ - Convert test coroutines to test functions. - - This supports asychronous tests transparently. - - """ - super().__init_subclass__(**kwargs) - for name in unittest.defaultTestLoader.getTestCaseNames(cls): - test = getattr(cls, name) - if asyncio.iscoroutinefunction(test): - setattr(cls, name, cls.convert_async_to_sync(test)) - - @staticmethod - def convert_async_to_sync(test): - """ - Convert a test coroutine to a test function. - - """ - - @functools.wraps(test) - def test_func(self, *args, **kwargs): - return self.loop.run_until_complete(test(self, *args, **kwargs)) - - return test_func - - def setUp(self): - super().setUp() - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - - def tearDown(self): - self.loop.close() - super().tearDown() - - def run_loop_once(self): - # Process callbacks scheduled with call_soon by appending a callback - # to stop the event loop then running it until it hits that callback. - self.loop.call_soon(self.loop.stop) - self.loop.run_forever() - - @contextlib.contextmanager - def assertNoLogs(self, logger="websockets", level=logging.ERROR): - """ - No message is logged on the given logger with at least the given level. - - """ - with self.assertLogs(logger, level) as logs: - # We want to test that no log message is emitted - # but assertLogs expects at least one log message. - logging.getLogger(logger).log(level, "dummy") - yield - - level_name = logging.getLevelName(level) - self.assertEqual(logs.output, [f"{level_name}:{logger}:dummy"]) - - def assertDeprecationWarnings(self, recorded_warnings, expected_warnings): - """ - Check recorded deprecation warnings match a list of expected messages. - - """ - self.assertEqual(len(recorded_warnings), len(expected_warnings)) - for recorded, expected in zip(recorded_warnings, expected_warnings): - actual = recorded.message - self.assertEqual(str(actual), expected) - self.assertEqual(type(actual), DeprecationWarning) - - -# Unit for timeouts. May be increased on slow machines by setting the -# WEBSOCKETS_TESTS_TIMEOUT_FACTOR environment variable. -MS = 0.001 * int(os.environ.get("WEBSOCKETS_TESTS_TIMEOUT_FACTOR", 1)) - -# asyncio's debug mode has a 10x performance penalty for this test suite. -if os.environ.get("PYTHONASYNCIODEBUG"): # pragma: no cover - MS *= 10 - -# Ensure that timeouts are larger than the clock's resolution (for Windows). -MS = max(MS, 2.5 * time.get_clock_info("monotonic").resolution) From 9a99229c671711d6274d3914244694e106966268 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 29 Nov 2020 15:45:41 +0100 Subject: [PATCH 062/104] Explain backwards-compatibility & versioning policies. --- docs/changelog.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 291ec6938..2d2e7ca08 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,23 @@ Changelog .. currentmodule:: websockets +Backwards-compatibility policy +.............................. + +``websockets`` is intended for production use. Therefore, stability is a goal. + +``websockets`` also aims at providing the best API for WebSocket in Python. + +While we value stability, we value progress more. When an improvement requires +changing the API, we make the change and document it below. + +When possible with reasonable effort, we preserve backwards-compatibility for +five years after the release that introduced the change. + +When a release contains backwards-incompatible API changes, the major version +is increased, else the minor version is increased. Patch versions are only for +fixing regressions shortly after a release. + 9.0 ... From 9c14a2f981af2da3517564ea7396ea06e19114d3 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 29 Nov 2020 18:02:12 +0100 Subject: [PATCH 063/104] Review and update changelog. * Add missing items for 9.0 release. * Re-assess infos / warnings. * Add release dates. --- docs/changelog.rst | 276 ++++++++++++++++++++++++++++----------------- 1 file changed, 174 insertions(+), 102 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2d2e7ca08..8d255fdfd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,36 +29,54 @@ fixing regressions shortly after a release. **Version 9.0 moves or deprecates several APIs.** - * Import :class:`~datastructures.Headers` and - :exc:`~datastructures.MultipleValuesError` from - :mod:`websockets.datastructures` instead of :mod:`websockets.http`. + * :class:`~datastructures.Headers` and + :exc:`~datastructures.MultipleValuesError` were moved from + ``websockets.http`` to :mod:`websockets.datastructures`. - * :mod:`websockets.client`, :mod:`websockets.server,` - :mod:`websockets.protocol`, and :mod:`websockets.auth` were moved to - :mod:`websockets.legacy.client`, :mod:`websockets.legacy.server`, - :mod:`websockets.legacy.protocol`, and :mod:`websockets.legacy.auth` - respectively. + * ``websockets.client``, ``websockets.server``, ``websockets.protocol``, + and ``websockets.auth`` were moved to :mod:`websockets.legacy.client`, + :mod:`websockets.legacy.server`, :mod:`websockets.legacy.protocol`, and + :mod:`websockets.legacy.auth` respectively. - * :mod:`websockets.handshake` is deprecated. + * ``websockets.handshake`` is deprecated. - * :mod:`websockets.http` is deprecated. + * ``websockets.http`` is deprecated. - * :mod:`websockets.framing` is deprecated. + * ``websockets.framing`` is deprecated. Aliases provide backwards compatibility for all previously public APIs. +* Added compatibility with Python 3.9. + * Added support for IRIs in addition to URIs. +* Added close codes 1012, 1013, and 1014. + +* Raised an error when passing a :class:`dict` to + :meth:`~legacy.protocol.WebSocketCommonProtocol.send`. + +* Fixed ``Host`` header sent when connecting to an IPv6 address. + +* Aligned maximum cookie size with popular web browsers. + +* Ensured cancellation always propagates, even on Python versions where + :exc:`~asyncio.CancelledError` inherits :exc:`Exception`. + +* Improved error reporting. + + 8.1 ... -* Added compatibility with Python 3.8. +*November 1, 2019* -* Added close codes 1012, 1013, and 1014. +* Added compatibility with Python 3.8. 8.0.2 ..... +*July 31, 2019* + * Restored the ability to pass a socket with the ``sock`` parameter of :func:`~legacy.server.serve`. @@ -67,12 +85,16 @@ fixing regressions shortly after a release. 8.0.1 ..... +*July 21, 2019* + * Restored the ability to import ``WebSocketProtocolError`` from ``websockets``. 8.0 ... +*July 7, 2019* + .. warning:: **Version 8.0 drops compatibility with Python 3.4 and 3.5.** @@ -83,7 +105,8 @@ fixing regressions shortly after a release. Previously, it could be a function or a coroutine. - If you're passing a ``process_request`` argument to :func:`~legacy.server.serve` + If you're passing a ``process_request`` argument to + :func:`~legacy.server.serve` or :class:`~legacy.server.WebSocketServerProtocol`, or if you're overriding :meth:`~legacy.server.WebSocketServerProtocol.process_request` in a subclass, define it with ``async def`` instead of ``def``. @@ -103,36 +126,38 @@ fixing regressions shortly after a release. **Version 8.0 deprecates the** ``host`` **,** ``port`` **, and** ``secure`` **attributes of** :class:`~legacy.protocol.WebSocketCommonProtocol`. - Use :attr:`~legacy.protocol.WebSocketCommonProtocol.local_address` in servers and + Use :attr:`~legacy.protocol.WebSocketCommonProtocol.local_address` in + servers and :attr:`~legacy.protocol.WebSocketCommonProtocol.remote_address` in clients instead of ``host`` and ``port``. .. note:: **Version 8.0 renames the** ``WebSocketProtocolError`` **exception** - to :exc:`ProtocolError` **.** + to :exc:`~exceptions.ProtocolError` **.** A ``WebSocketProtocolError`` alias provides backwards compatibility. .. note:: **Version 8.0 adds the reason phrase to the return type of the low-level - API** :func:`~http.read_response` **.** + API** ``read_response()`` **.** Also: * :meth:`~legacy.protocol.WebSocketCommonProtocol.send`, :meth:`~legacy.protocol.WebSocketCommonProtocol.ping`, and - :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` support bytes-like types - :class:`bytearray` and :class:`memoryview` in addition to :class:`bytes`. + :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` support bytes-like + types :class:`bytearray` and :class:`memoryview` in addition to + :class:`bytes`. * Added :exc:`~exceptions.ConnectionClosedOK` and :exc:`~exceptions.ConnectionClosedError` subclasses of :exc:`~exceptions.ConnectionClosed` to tell apart normal connection termination from errors. -* Added :func:`~legacy.auth.basic_auth_protocol_factory` to enforce HTTP Basic Auth - on the server side. +* Added :func:`~legacy.auth.basic_auth_protocol_factory` to enforce HTTP + Basic Auth on the server side. * :func:`~legacy.client.connect` handles redirects from the server during the handshake. @@ -148,8 +173,9 @@ Also: exceptions in keepalive ping task. If you were using ``ping_timeout=None`` as a workaround, you can remove it. -* Changed :meth:`WebSocketServer.close() ` to - perform a proper closing handshake instead of failing the connection. +* Changed :meth:`WebSocketServer.close() + ` to perform a proper closing handshake + instead of failing the connection. * Avoided a crash when a ``extra_headers`` callable returns ``None``. @@ -170,20 +196,20 @@ Also: 7.0 ... -.. warning:: +*November 1, 2018* - **Version 7.0 renames the** ``timeout`` **argument of** - :func:`~legacy.server.serve()` **and** :func:`~legacy.client.connect` **to** - ``close_timeout`` **.** +.. warning:: - This prevents confusion with ``ping_timeout``. + ``websockets`` **now sends Ping frames at regular intervals and closes the + connection if it doesn't receive a matching Pong frame.** - For backwards compatibility, ``timeout`` is still supported. + See :class:`~legacy.protocol.WebSocketCommonProtocol` for details. .. warning:: - **Version 7.0 changes how a server terminates connections when it's - closed with** :meth:`~legacy.server.WebSocketServer.close` **.** + **Version 7.0 changes how a server terminates connections when it's closed + with** :meth:`WebSocketServer.close() + ` **.** Previously, connections handlers were canceled. Now, connections are closed with close code 1001 (going away). From the perspective of the @@ -200,8 +226,19 @@ Also: .. note:: - **Version 7.0 changes how a** :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` - **that hasn't received a pong yet behaves when the connection is closed.** + **Version 7.0 renames the** ``timeout`` **argument of** + :func:`~legacy.server.serve` **and** :func:`~legacy.client.connect` **to** + ``close_timeout`` **.** + + This prevents confusion with ``ping_timeout``. + + For backwards compatibility, ``timeout`` is still supported. + +.. note:: + + **Version 7.0 changes how a** + :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` **that hasn't + received a pong yet behaves when the connection is closed.** The ping — as in ``ping = await websocket.ping()`` — used to be canceled when the connection is closed, so that ``await ping`` raised @@ -211,34 +248,33 @@ Also: .. note:: **Version 7.0 raises a** :exc:`RuntimeError` **exception if two coroutines - call** :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` **concurrently.** + call** :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` + **concurrently.** Concurrent calls lead to non-deterministic behavior because there are no guarantees about which coroutine will receive which message. Also: -* ``websockets`` sends Ping frames at regular intervals and closes the - connection if it doesn't receive a matching Pong frame. See - :class:`~legacy.protocol.WebSocketCommonProtocol` for details. - * Added ``process_request`` and ``select_subprotocol`` arguments to - :func:`~legacy.server.serve` and :class:`~legacy.server.WebSocketServerProtocol` to - customize :meth:`~legacy.server.WebSocketServerProtocol.process_request` and + :func:`~legacy.server.serve` and + :class:`~legacy.server.WebSocketServerProtocol` to customize + :meth:`~legacy.server.WebSocketServerProtocol.process_request` and :meth:`~legacy.server.WebSocketServerProtocol.select_subprotocol` without subclassing :class:`~legacy.server.WebSocketServerProtocol`. * Added support for sending fragmented messages. -* Added the :meth:`~legacy.protocol.WebSocketCommonProtocol.wait_closed` method to - protocols. +* Added the :meth:`~legacy.protocol.WebSocketCommonProtocol.wait_closed` + method to protocols. * Added an interactive client: ``python -m websockets ``. * Changed the ``origins`` argument to represent the lack of an origin with ``None`` rather than ``''``. -* Fixed a data loss bug in :meth:`~legacy.protocol.WebSocketCommonProtocol.recv`: +* Fixed a data loss bug in + :meth:`~legacy.protocol.WebSocketCommonProtocol.recv`: canceling it at the wrong time could result in messages being dropped. * Improved handling of multiple HTTP headers with the same name. @@ -248,36 +284,37 @@ Also: 6.0 ... +*July 16, 2018* + .. warning:: - **Version 6.0 introduces the** :class:`~http.Headers` **class for managing - HTTP headers and changes several public APIs:** + **Version 6.0 introduces the** :class:`~datastructures.Headers` **class + for managing HTTP headers and changes several public APIs:** - * :meth:`~legacy.server.WebSocketServerProtocol.process_request` now receives a - :class:`~http.Headers` instead of a :class:`~http.client.HTTPMessage` in - the ``request_headers`` argument. + * :meth:`~legacy.server.WebSocketServerProtocol.process_request` now + receives a :class:`~datastructures.Headers` instead of a + ``http.client.HTTPMessage`` in the ``request_headers`` argument. - * The :attr:`~legacy.protocol.WebSocketCommonProtocol.request_headers` and - :attr:`~legacy.protocol.WebSocketCommonProtocol.response_headers` attributes of - :class:`~legacy.protocol.WebSocketCommonProtocol` are :class:`~http.Headers` - instead of :class:`~http.client.HTTPMessage`. + * The ``request_headers`` and ``response_headers`` attributes of + :class:`~legacy.protocol.WebSocketCommonProtocol` are + :class:`~datastructures.Headers` instead of ``http.client.HTTPMessage``. - * The :attr:`~legacy.protocol.WebSocketCommonProtocol.raw_request_headers` and - :attr:`~legacy.protocol.WebSocketCommonProtocol.raw_response_headers` - attributes of :class:`~legacy.protocol.WebSocketCommonProtocol` are removed. - Use :meth:`~http.Headers.raw_items` instead. + * The ``raw_request_headers`` and ``raw_response_headers`` attributes of + :class:`~legacy.protocol.WebSocketCommonProtocol` are removed. Use + :meth:`~datastructures.Headers.raw_items` instead. - * Functions defined in the :mod:`~handshake` module now receive - :class:`~http.Headers` in argument instead of ``get_header`` or - ``set_header`` functions. This affects libraries that rely on + * Functions defined in the ``handshake`` module now receive + :class:`~datastructures.Headers` in argument instead of ``get_header`` + or ``set_header`` functions. This affects libraries that rely on low-level APIs. - * Functions defined in the :mod:`~http` module now return HTTP headers as - :class:`~http.Headers` instead of lists of ``(name, value)`` pairs. + * Functions defined in the ``http`` module now return HTTP headers as + :class:`~datastructures.Headers` instead of lists of ``(name, value)`` + pairs. - Since :class:`~http.Headers` and :class:`~http.client.HTTPMessage` provide - similar APIs, this change won't affect most of the code dealing with HTTP - headers. + Since :class:`~datastructures.Headers` and ``http.client.HTTPMessage`` + provide similar APIs, this change won't affect most of the code dealing + with HTTP headers. Also: @@ -287,12 +324,16 @@ Also: 5.0.1 ..... -* Fixed a regression in the 5.0 release that broke some invocations of - :func:`~legacy.server.serve()` and :func:`~legacy.client.connect`. +*May 24, 2018* + +* Fixed a regression in 5.0 that broke some invocations of + :func:`~legacy.server.serve` and :func:`~legacy.client.connect`. 5.0 ... +*May 22, 2018* + .. note:: **Version 5.0 fixes a security issue introduced in version 4.0.** @@ -308,8 +349,8 @@ Also: **Version 5.0 adds a** ``user_info`` **field to the return value of** :func:`~uri.parse_uri` **and** :class:`~uri.WebSocketURI` **.** - If you're unpacking :class:`~exceptions.WebSocketURI` into four variables, - adjust your code to account for that fifth field. + If you're unpacking :class:`~uri.WebSocketURI` into four variables, adjust + your code to account for that fifth field. Also: @@ -322,14 +363,14 @@ Also: * A plain HTTP request now receives a 426 Upgrade Required response and doesn't log a stack trace. -* :func:`~legacy.server.unix_serve` can be used as an asynchronous context manager on - Python ≥ 3.5.1. +* :func:`~legacy.server.unix_serve` can be used as an asynchronous context + manager on Python ≥ 3.5.1. -* Added the :attr:`~legacy.protocol.WebSocketCommonProtocol.closed` property to - protocols. +* Added the :attr:`~legacy.protocol.WebSocketCommonProtocol.closed` property + to protocols. -* If a :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` doesn't receive a pong, - it's canceled when the connection is closed. +* If a :meth:`~legacy.protocol.WebSocketCommonProtocol.ping` doesn't receive a + pong, it's canceled when the connection is closed. * Reported the cause of :exc:`~exceptions.ConnectionClosed` exceptions. @@ -355,13 +396,21 @@ Also: 4.0.1 ..... +*November 2, 2017* + * Fixed issues with the packaging of the 4.0 release. 4.0 ... +*November 2, 2017* + .. warning:: + **Version 4.0 drops compatibility with Python 3.3.** + +.. note:: + **Version 4.0 enables compression with the permessage-deflate extension.** In August 2017, Firefox and Chrome support it, but not Safari and IE. @@ -369,11 +418,7 @@ Also: Compression should improve performance but it increases RAM and CPU use. If you want to disable compression, add ``compression=None`` when calling - :func:`~legacy.server.serve()` or :func:`~legacy.client.connect`. - -.. warning:: - - **Version 4.0 drops compatibility with Python 3.3.** + :func:`~legacy.server.serve` or :func:`~legacy.client.connect`. .. note:: @@ -388,8 +433,8 @@ Also: * Added :func:`~legacy.server.unix_serve` for listening on Unix sockets. -* Added the :attr:`~legacy.server.WebSocketServer.sockets` attribute to the return - value of :func:`~legacy.server.serve`. +* Added the :attr:`~legacy.server.WebSocketServer.sockets` attribute to the + return value of :func:`~legacy.server.serve`. * Reorganized and extended documentation. @@ -407,12 +452,14 @@ Also: 3.4 ... -* Renamed :func:`~legacy.server.serve()` and :func:`~legacy.client.connect`'s ``klass`` - argument to ``create_protocol`` to reflect that it can also be a callable. - For backwards compatibility, ``klass`` is still supported. +*August 20, 2017* + +* Renamed :func:`~legacy.server.serve` and :func:`~legacy.client.connect`'s + ``klass`` argument to ``create_protocol`` to reflect that it can also be a + callable. For backwards compatibility, ``klass`` is still supported. -* :func:`~legacy.server.serve` can be used as an asynchronous context manager on - Python ≥ 3.5.1. +* :func:`~legacy.server.serve` can be used as an asynchronous context manager + on Python ≥ 3.5.1. * Added support for customizing handling of incoming connections with :meth:`~legacy.server.WebSocketServerProtocol.process_request`. @@ -423,8 +470,8 @@ Also: * Added an optional C extension to speed up low-level operations. -* An invalid response status code during :func:`~legacy.client.connect` now raises - :class:`~exceptions.InvalidStatusCode` with a ``code`` attribute. +* An invalid response status code during :func:`~legacy.client.connect` now + raises :class:`~exceptions.InvalidStatusCode` with a ``code`` attribute. * Providing a ``sock`` argument to :func:`~legacy.client.connect` no longer crashes. @@ -432,6 +479,8 @@ Also: 3.3 ... +*March 29, 2017* + * Ensured compatibility with Python 3.6. * Reduced noise in logs caused by connection resets. @@ -441,14 +490,18 @@ Also: 3.2 ... +*August 17, 2016* + * Added ``timeout``, ``max_size``, and ``max_queue`` arguments to - :func:`~legacy.client.connect()` and :func:`~legacy.server.serve`. + :func:`~legacy.client.connect` and :func:`~legacy.server.serve`. * Made server shutdown more robust. 3.1 ... +*April 21, 2016* + * Avoided a warning when closing a connection before the opening handshake. * Added flow control for incoming data. @@ -456,6 +509,8 @@ Also: 3.0 ... +*December 25, 2015* + .. warning:: **Version 3.0 introduces a backwards-incompatible change in the** @@ -463,9 +518,9 @@ Also: **If you're upgrading from 2.x or earlier, please read this carefully.** - :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` used to return ``None`` - when the connection was closed. This required checking the return value of - every call:: + :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` used to return + ``None`` when the connection was closed. This required checking the return + value of every call:: message = await websocket.recv() if message is None: @@ -484,13 +539,13 @@ Also: previous behavior can be restored by passing ``legacy_recv=True`` to :func:`~legacy.server.serve`, :func:`~legacy.client.connect`, :class:`~legacy.server.WebSocketServerProtocol`, or - :class:`~legacy.client.WebSocketClientProtocol`. ``legacy_recv`` isn't documented - in their signatures but isn't scheduled for deprecation either. + :class:`~legacy.client.WebSocketClientProtocol`. ``legacy_recv`` isn't + documented in their signatures but isn't scheduled for deprecation either. Also: -* :func:`~legacy.client.connect` can be used as an asynchronous context manager on - Python ≥ 3.5.1. +* :func:`~legacy.client.connect` can be used as an asynchronous context + manager on Python ≥ 3.5.1. * Updated documentation with ``await`` and ``async`` syntax from Python 3.5. @@ -498,7 +553,8 @@ Also: :meth:`~legacy.protocol.WebSocketCommonProtocol.pong` support data passed as :class:`str` in addition to :class:`bytes`. -* Worked around an asyncio bug affecting connection termination under load. +* Worked around an :mod:`asyncio` bug affecting connection termination under + load. * Made ``state_name`` attribute on protocols a public API. @@ -507,6 +563,8 @@ Also: 2.7 ... +*November 18, 2015* + * Added compatibility with Python 3.5. * Refreshed documentation. @@ -514,6 +572,8 @@ Also: 2.6 ... +*August 18, 2015* + * Added ``local_address`` and ``remote_address`` attributes on protocols. * Closed open connections with code 1001 when a server shuts down. @@ -523,19 +583,21 @@ Also: 2.5 ... +*July 28, 2015* + * Improved documentation. * Provided access to handshake request and response HTTP headers. * Allowed customizing handshake request and response HTTP headers. -* Supported running on a non-default event loop. +* Added support for running on a non-default event loop. * Returned a 403 status code instead of 400 when the request Origin isn't allowed. -* Canceling :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` no longer drops - the next message. +* Canceling :meth:`~legacy.protocol.WebSocketCommonProtocol.recv` no longer + drops the next message. * Clarified that the closing handshake can be initiated by the client. @@ -548,9 +610,9 @@ Also: 2.4 ... -* Added support for subprotocols. +*January 31, 2015* -* Supported non-default event loop. +* Added support for subprotocols. * Added ``loop`` argument to :func:`~legacy.client.connect` and :func:`~legacy.server.serve`. @@ -558,16 +620,22 @@ Also: 2.3 ... +*November 3, 2014* + * Improved compliance of close codes. 2.2 ... +*July 28, 2014* + * Added support for limiting message size. 2.1 ... +*April 26, 2014* + * Added ``host``, ``port`` and ``secure`` attributes on protocols. * Added support for providing and checking Origin_. @@ -577,6 +645,8 @@ Also: 2.0 ... +*February 16, 2014* + .. warning:: **Version 2.0 introduces a backwards-incompatible change in the** @@ -603,4 +673,6 @@ Also: 1.0 ... +*November 14, 2013* + * Initial public release. From 94256f4f41ef024f7f511a573763bd755f5f1b46 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 29 Nov 2020 18:02:27 +0100 Subject: [PATCH 064/104] Update word list for spell check. --- docs/spelling_wordlist.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 1eacc491d..dd3500b73 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -5,14 +5,21 @@ awaitable aymeric backpressure Backpressure +balancer +balancers Bitcoin +bottlenecked bufferbloat Bufferbloat bugfix bytestring bytestrings changelog +coroutine +coroutines +cryptocurrencies cryptocurrency +Ctrl daemonize fractalideas iterable @@ -20,18 +27,25 @@ keepalive KiB lifecycle Lifecycle +lookups MiB nginx +parsers permessage pong pongs Pythonic serializers +Subclasses +subclasses subclassing subprotocol subprotocols +Tidelift TLS +tox Unparse +unregister uple username websocket From 42f0e2c0b8e994c33b792208adff32bea1cdff4f Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 29 Nov 2020 22:01:42 +0100 Subject: [PATCH 065/104] Add helper to manage aliases and deprecations. This may save a little bit of CPU and memory by avoiding unnecessary imports too, especially as the library grows. --- src/websockets/__init__.py | 72 +++++++++++++++++++++++----- src/websockets/auth.py | 4 -- src/websockets/client.py | 11 ++++- src/websockets/framing.py | 6 --- src/websockets/handshake.py | 45 ------------------ src/websockets/http.py | 45 +++++++----------- src/websockets/imports.py | 95 +++++++++++++++++++++++++++++++++++++ src/websockets/protocol.py | 1 - src/websockets/server.py | 18 ++++--- tests/test_auth.py | 2 - tests/test_exports.py | 9 ++++ tests/test_framing.py | 9 ---- tests/test_handshake.py | 2 - tests/test_imports.py | 53 +++++++++++++++++++++ tests/test_protocol.py | 2 - 15 files changed, 256 insertions(+), 118 deletions(-) delete mode 100644 src/websockets/auth.py delete mode 100644 src/websockets/framing.py delete mode 100644 src/websockets/handshake.py create mode 100644 src/websockets/imports.py delete mode 100644 src/websockets/protocol.py delete mode 100644 tests/test_auth.py delete mode 100644 tests/test_framing.py delete mode 100644 tests/test_handshake.py create mode 100644 tests/test_imports.py delete mode 100644 tests/test_protocol.py diff --git a/src/websockets/__init__.py b/src/websockets/__init__.py index 0242e7942..580a3960f 100644 --- a/src/websockets/__init__.py +++ b/src/websockets/__init__.py @@ -1,19 +1,8 @@ -# This relies on each of the submodules having an __all__ variable. - -from .client import * -from .datastructures import * # noqa -from .exceptions import * # noqa -from .legacy.auth import * # noqa -from .legacy.client import * # noqa -from .legacy.protocol import * # noqa -from .legacy.server import * # noqa -from .server import * -from .typing import * # noqa -from .uri import * # noqa +from .imports import lazy_import from .version import version as __version__ # noqa -__all__ = [ +__all__ = [ # noqa "AbortHandshake", "basic_auth_protocol_factory", "BasicAuthWebSocketServerProtocol", @@ -58,3 +47,60 @@ "WebSocketServerProtocol", "WebSocketURI", ] + +lazy_import( + globals(), + aliases={ + "auth": ".legacy", + "basic_auth_protocol_factory": ".legacy.auth", + "BasicAuthWebSocketServerProtocol": ".legacy.auth", + "ClientConnection": ".client", + "connect": ".legacy.client", + "unix_connect": ".legacy.client", + "WebSocketClientProtocol": ".legacy.client", + "Headers": ".datastructures", + "MultipleValuesError": ".datastructures", + "WebSocketException": ".exceptions", + "ConnectionClosed": ".exceptions", + "ConnectionClosedError": ".exceptions", + "ConnectionClosedOK": ".exceptions", + "InvalidHandshake": ".exceptions", + "SecurityError": ".exceptions", + "InvalidMessage": ".exceptions", + "InvalidHeader": ".exceptions", + "InvalidHeaderFormat": ".exceptions", + "InvalidHeaderValue": ".exceptions", + "InvalidOrigin": ".exceptions", + "InvalidUpgrade": ".exceptions", + "InvalidStatusCode": ".exceptions", + "NegotiationError": ".exceptions", + "DuplicateParameter": ".exceptions", + "InvalidParameterName": ".exceptions", + "InvalidParameterValue": ".exceptions", + "AbortHandshake": ".exceptions", + "RedirectHandshake": ".exceptions", + "InvalidState": ".exceptions", + "InvalidURI": ".exceptions", + "PayloadTooBig": ".exceptions", + "ProtocolError": ".exceptions", + "WebSocketProtocolError": ".exceptions", + "protocol": ".legacy", + "WebSocketCommonProtocol": ".legacy.protocol", + "ServerConnection": ".server", + "serve": ".legacy.server", + "unix_serve": ".legacy.server", + "WebSocketServerProtocol": ".legacy.server", + "WebSocketServer": ".legacy.server", + "Data": ".typing", + "Origin": ".typing", + "ExtensionHeader": ".typing", + "ExtensionParameter": ".typing", + "Subprotocol": ".typing", + "parse_uri": ".uri", + "WebSocketURI": ".uri", + }, + deprecated_aliases={ + "framing": ".legacy", + "handshake": ".legacy", + }, +) diff --git a/src/websockets/auth.py b/src/websockets/auth.py deleted file mode 100644 index c8839c401..000000000 --- a/src/websockets/auth.py +++ /dev/null @@ -1,4 +0,0 @@ -from .legacy.auth import BasicAuthWebSocketServerProtocol, basic_auth_protocol_factory - - -__all__ = ["BasicAuthWebSocketServerProtocol", "basic_auth_protocol_factory"] diff --git a/src/websockets/client.py b/src/websockets/client.py index 8cababed5..91dd1662e 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -24,7 +24,7 @@ ) from .http import USER_AGENT, build_host from .http11 import Request, Response -from .legacy.client import WebSocketClientProtocol, connect, unix_connect # noqa +from .imports import lazy_import from .typing import ( ConnectionOption, ExtensionHeader, @@ -36,6 +36,15 @@ from .utils import accept_key, generate_key +lazy_import( + globals(), + aliases={ + "connect": ".legacy.client", + "unix_connect": ".legacy.client", + "WebSocketClientProtocol": ".legacy.client", + }, +) + __all__ = ["ClientConnection"] logger = logging.getLogger(__name__) diff --git a/src/websockets/framing.py b/src/websockets/framing.py deleted file mode 100644 index 2dadb5610..000000000 --- a/src/websockets/framing.py +++ /dev/null @@ -1,6 +0,0 @@ -import warnings - -from .legacy.framing import * # noqa - - -warnings.warn("websockets.framing is deprecated", DeprecationWarning) diff --git a/src/websockets/handshake.py b/src/websockets/handshake.py deleted file mode 100644 index cc4010d41..000000000 --- a/src/websockets/handshake.py +++ /dev/null @@ -1,45 +0,0 @@ -import warnings - -from .datastructures import Headers - - -__all__ = ["build_request", "check_request", "build_response", "check_response"] - - -# Backwards compatibility with previously documented public APIs - - -def build_request(headers: Headers) -> str: # pragma: no cover - warnings.warn( - "websockets.handshake.build_request is deprecated", DeprecationWarning - ) - from .legacy.handshake import build_request - - return build_request(headers) - - -def check_request(headers: Headers) -> str: # pragma: no cover - warnings.warn( - "websockets.handshake.check_request is deprecated", DeprecationWarning - ) - from .legacy.handshake import check_request - - return check_request(headers) - - -def build_response(headers: Headers, key: str) -> None: # pragma: no cover - warnings.warn( - "websockets.handshake.build_response is deprecated", DeprecationWarning - ) - from .legacy.handshake import build_response - - return build_response(headers, key) - - -def check_response(headers: Headers, key: str) -> None: # pragma: no cover - warnings.warn( - "websockets.handshake.check_response is deprecated", DeprecationWarning - ) - from .legacy.handshake import check_response - - return check_response(headers, key) diff --git a/src/websockets/http.py b/src/websockets/http.py index b05b78455..9092836c2 100644 --- a/src/websockets/http.py +++ b/src/websockets/http.py @@ -1,15 +1,27 @@ -import asyncio import ipaddress import sys -import warnings -from typing import Tuple -# For backwards compatibility: -# Headers and MultipleValuesError used to be defined in this module -from .datastructures import Headers, MultipleValuesError # noqa +from .imports import lazy_import from .version import version as websockets_version +# For backwards compatibility: + + +lazy_import( + globals(), + # Headers and MultipleValuesError used to be defined in this module. + aliases={ + "Headers": ".datastructures", + "MultipleValuesError": ".datastructures", + }, + deprecated_aliases={ + "read_request": ".legacy.http", + "read_response": ".legacy.http", + }, +) + + __all__ = ["USER_AGENT", "build_host"] @@ -38,24 +50,3 @@ def build_host(host: str, port: int, secure: bool) -> str: host = f"{host}:{port}" return host - - -# Backwards compatibility with previously documented public APIs - - -async def read_request( - stream: asyncio.StreamReader, -) -> Tuple[str, Headers]: # pragma: no cover - warnings.warn("websockets.http.read_request is deprecated", DeprecationWarning) - from .legacy.http import read_request - - return await read_request(stream) - - -async def read_response( - stream: asyncio.StreamReader, -) -> Tuple[int, str, Headers]: # pragma: no cover - warnings.warn("websockets.http.read_response is deprecated", DeprecationWarning) - from .legacy.http import read_response - - return await read_response(stream) diff --git a/src/websockets/imports.py b/src/websockets/imports.py new file mode 100644 index 000000000..9a4cfd98a --- /dev/null +++ b/src/websockets/imports.py @@ -0,0 +1,95 @@ +import importlib +import sys +import warnings +from typing import Any, Dict, Iterable, Optional + + +__all__ = ["lazy_import"] + + +def lazy_import( + namespace: Dict[str, Any], + aliases: Optional[Dict[str, str]] = None, + deprecated_aliases: Optional[Dict[str, str]] = None, +) -> None: + """ + Provide lazy, module-level imports. + + Typical use:: + + __getattr__, __dir__ = lazy_import( + globals(), + aliases={ + "": "", + ... + }, + deprecated_aliases={ + ..., + } + ) + + This function defines __getattr__ and __dir__ per PEP 562. + + On Python 3.6 and earlier, it falls back to non-lazy imports and doesn't + raise deprecation warnings. + + """ + if aliases is None: + aliases = {} + if deprecated_aliases is None: + deprecated_aliases = {} + + namespace_set = set(namespace) + aliases_set = set(aliases) + deprecated_aliases_set = set(deprecated_aliases) + + assert not namespace_set & aliases_set, "namespace conflict" + assert not namespace_set & deprecated_aliases_set, "namespace conflict" + assert not aliases_set & deprecated_aliases_set, "namespace conflict" + + package = namespace["__name__"] + + if sys.version_info[:2] >= (3, 7): + + def __getattr__(name: str) -> Any: + assert aliases is not None # mypy cannot figure this out + try: + source = aliases[name] + except KeyError: + pass + else: + module = importlib.import_module(source, package) + return getattr(module, name) + + assert deprecated_aliases is not None # mypy cannot figure this out + try: + source = deprecated_aliases[name] + except KeyError: + pass + else: + warnings.warn( + f"{package}.{name} is deprecated", + DeprecationWarning, + stacklevel=2, + ) + module = importlib.import_module(source, package) + return getattr(module, name) + + raise AttributeError(f"module {package!r} has no attribute {name!r}") + + namespace["__getattr__"] = __getattr__ + + def __dir__() -> Iterable[str]: + return sorted(namespace_set | aliases_set | deprecated_aliases_set) + + namespace["__dir__"] = __dir__ + + else: # pragma: no cover + + for name, source in aliases.items(): + module = importlib.import_module(source, package) + namespace[name] = getattr(module, name) + + for name, source in deprecated_aliases.items(): + module = importlib.import_module(source, package) + namespace[name] = getattr(module, name) diff --git a/src/websockets/protocol.py b/src/websockets/protocol.py deleted file mode 100644 index 287f92a57..000000000 --- a/src/websockets/protocol.py +++ /dev/null @@ -1 +0,0 @@ -from .legacy.protocol import * # noqa diff --git a/src/websockets/server.py b/src/websockets/server.py index bd527be74..67ab83031 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -26,12 +26,7 @@ ) from .http import USER_AGENT from .http11 import Request, Response -from .legacy.server import ( # noqa - WebSocketServer, - WebSocketServerProtocol, - serve, - unix_serve, -) +from .imports import lazy_import from .typing import ( ConnectionOption, ExtensionHeader, @@ -42,6 +37,17 @@ from .utils import accept_key +lazy_import( + globals(), + aliases={ + "serve": ".legacy.server", + "unix_serve": ".legacy.server", + "WebSocketServerProtocol": ".legacy.server", + "WebSocketServer": ".legacy.server", + }, +) + + __all__ = ["ServerConnection"] logger = logging.getLogger(__name__) diff --git a/tests/test_auth.py b/tests/test_auth.py deleted file mode 100644 index 01ca207c7..000000000 --- a/tests/test_auth.py +++ /dev/null @@ -1,2 +0,0 @@ -# Check that the legacy auth module imports without an exception. -from websockets.auth import * # noqa diff --git a/tests/test_exports.py b/tests/test_exports.py index 8e4330304..568c50c54 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -1,6 +1,15 @@ import unittest import websockets +import websockets.client +import websockets.exceptions +import websockets.legacy.auth +import websockets.legacy.client +import websockets.legacy.protocol +import websockets.legacy.server +import websockets.server +import websockets.typing +import websockets.uri combined_exports = ( diff --git a/tests/test_framing.py b/tests/test_framing.py deleted file mode 100644 index d6fa6352a..000000000 --- a/tests/test_framing.py +++ /dev/null @@ -1,9 +0,0 @@ -import warnings - - -with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "websockets.framing is deprecated", DeprecationWarning - ) - # Check that the legacy framing module imports without an exception. - from websockets.framing import * # noqa diff --git a/tests/test_handshake.py b/tests/test_handshake.py deleted file mode 100644 index 8c35c9714..000000000 --- a/tests/test_handshake.py +++ /dev/null @@ -1,2 +0,0 @@ -# Check that the legacy handshake module imports without an exception. -from websockets.handshake import * # noqa diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 000000000..113564e9f --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,53 @@ +import types +import unittest +import warnings + +from websockets.imports import * + + +foo = object() + +bar = object() + + +class ImportsTests(unittest.TestCase): + def test_get_alias(self): + mod = types.ModuleType("tests.test_imports.test_alias") + lazy_import(vars(mod), aliases={"foo": ".."}) + + self.assertEqual(mod.foo, foo) + + def test_get_deprecated_alias(self): + mod = types.ModuleType("tests.test_imports.test_alias") + lazy_import(vars(mod), deprecated_aliases={"bar": ".."}) + + with warnings.catch_warnings(record=True) as recorded_warnings: + self.assertEqual(mod.bar, bar) + + self.assertEqual(len(recorded_warnings), 1) + warning = recorded_warnings[0].message + self.assertEqual( + str(warning), "tests.test_imports.test_alias.bar is deprecated" + ) + self.assertEqual(type(warning), DeprecationWarning) + + def test_dir(self): + mod = types.ModuleType("tests.test_imports.test_alias") + lazy_import(vars(mod), aliases={"foo": ".."}, deprecated_aliases={"bar": ".."}) + + self.assertEqual( + [item for item in dir(mod) if not item[:2] == item[-2:] == "__"], + ["bar", "foo"], + ) + + def test_attribute_error(self): + mod = types.ModuleType("tests.test_imports.test_alias") + lazy_import(vars(mod)) + + with self.assertRaises(AttributeError) as raised: + mod.foo + + self.assertEqual( + str(raised.exception), + "module 'tests.test_imports.test_alias' has no attribute 'foo'", + ) diff --git a/tests/test_protocol.py b/tests/test_protocol.py deleted file mode 100644 index f896fcae4..000000000 --- a/tests/test_protocol.py +++ /dev/null @@ -1,2 +0,0 @@ -# Check that the legacy protocol module imports without an exception. -from websockets.protocol import * # noqa From 965f8ec77347adaaf23c82eef693c9882269b46c Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 30 Nov 2020 21:46:25 +0100 Subject: [PATCH 066/104] Fix lazy imports of objects on Python 3.6. --- src/websockets/imports.py | 34 +++++++++++++++++++++++++--------- tests/test_imports.py | 39 +++++++++++++++++++++++++++------------ 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/src/websockets/imports.py b/src/websockets/imports.py index 9a4cfd98a..efd3eabf3 100644 --- a/src/websockets/imports.py +++ b/src/websockets/imports.py @@ -1,4 +1,3 @@ -import importlib import sys import warnings from typing import Any, Dict, Iterable, Optional @@ -7,6 +6,27 @@ __all__ = ["lazy_import"] +def import_name(name: str, source: str, namespace: Dict[str, Any]) -> Any: + """ + Import from in . + + There are two cases: + + - is an object defined in + - is a submodule of source + + Neither __import__ nor importlib.import_module does exactly this. + __import__ is closer to the intended behavior. + + """ + level = 0 + while source[level] == ".": + level += 1 + assert level < len(source), "importing from parent isn't supported" + module = __import__(source[level:], namespace, None, [name], level) + return getattr(module, name) + + def lazy_import( namespace: Dict[str, Any], aliases: Optional[Dict[str, str]] = None, @@ -58,8 +78,7 @@ def __getattr__(name: str) -> Any: except KeyError: pass else: - module = importlib.import_module(source, package) - return getattr(module, name) + return import_name(name, source, namespace) assert deprecated_aliases is not None # mypy cannot figure this out try: @@ -72,8 +91,7 @@ def __getattr__(name: str) -> Any: DeprecationWarning, stacklevel=2, ) - module = importlib.import_module(source, package) - return getattr(module, name) + return import_name(name, source, namespace) raise AttributeError(f"module {package!r} has no attribute {name!r}") @@ -87,9 +105,7 @@ def __dir__() -> Iterable[str]: else: # pragma: no cover for name, source in aliases.items(): - module = importlib.import_module(source, package) - namespace[name] = getattr(module, name) + namespace[name] = import_name(name, source, namespace) for name, source in deprecated_aliases.items(): - module = importlib.import_module(source, package) - namespace[name] = getattr(module, name) + namespace[name] = import_name(name, source, namespace) diff --git a/tests/test_imports.py b/tests/test_imports.py index 113564e9f..d84808902 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -1,3 +1,4 @@ +import sys import types import unittest import warnings @@ -11,18 +12,30 @@ class ImportsTests(unittest.TestCase): + def setUp(self): + self.mod = types.ModuleType("tests.test_imports.test_alias") + self.mod.__package__ = self.mod.__name__ + def test_get_alias(self): - mod = types.ModuleType("tests.test_imports.test_alias") - lazy_import(vars(mod), aliases={"foo": ".."}) + lazy_import( + vars(self.mod), + aliases={"foo": "...test_imports"}, + ) - self.assertEqual(mod.foo, foo) + self.assertEqual(self.mod.foo, foo) def test_get_deprecated_alias(self): - mod = types.ModuleType("tests.test_imports.test_alias") - lazy_import(vars(mod), deprecated_aliases={"bar": ".."}) + lazy_import( + vars(self.mod), + deprecated_aliases={"bar": "...test_imports"}, + ) with warnings.catch_warnings(record=True) as recorded_warnings: - self.assertEqual(mod.bar, bar) + self.assertEqual(self.mod.bar, bar) + + # No warnings raised on pre-PEP 526 Python. + if sys.version_info[:2] < (3, 7): # pragma: no cover + return self.assertEqual(len(recorded_warnings), 1) warning = recorded_warnings[0].message @@ -32,20 +45,22 @@ def test_get_deprecated_alias(self): self.assertEqual(type(warning), DeprecationWarning) def test_dir(self): - mod = types.ModuleType("tests.test_imports.test_alias") - lazy_import(vars(mod), aliases={"foo": ".."}, deprecated_aliases={"bar": ".."}) + lazy_import( + vars(self.mod), + aliases={"foo": "...test_imports"}, + deprecated_aliases={"bar": "...test_imports"}, + ) self.assertEqual( - [item for item in dir(mod) if not item[:2] == item[-2:] == "__"], + [item for item in dir(self.mod) if not item[:2] == item[-2:] == "__"], ["bar", "foo"], ) def test_attribute_error(self): - mod = types.ModuleType("tests.test_imports.test_alias") - lazy_import(vars(mod)) + lazy_import(vars(self.mod)) with self.assertRaises(AttributeError) as raised: - mod.foo + self.mod.foo self.assertEqual( str(raised.exception), From ecf64e7a56ee85e10a812139a4aee09e736aa241 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 30 Nov 2020 22:36:21 +0100 Subject: [PATCH 067/104] Handle non-contiguous memoryviews in C extension. This avoids the special-case in Python code. --- src/websockets/frames.py | 11 ++------ src/websockets/speedups.c | 51 ++++++++++++++++++----------------- tests/legacy/test_protocol.py | 30 --------------------- tests/test_frames.py | 9 ------- tests/test_utils.py | 24 +++-------------- 5 files changed, 32 insertions(+), 93 deletions(-) diff --git a/src/websockets/frames.py b/src/websockets/frames.py index 74223c0e8..71783e176 100644 --- a/src/websockets/frames.py +++ b/src/websockets/frames.py @@ -263,13 +263,8 @@ def prepare_data(data: Data) -> Tuple[int, bytes]: """ if isinstance(data, str): return OP_TEXT, data.encode("utf-8") - elif isinstance(data, (bytes, bytearray)): + elif isinstance(data, (bytes, bytearray, memoryview)): return OP_BINARY, data - elif isinstance(data, memoryview): - if data.c_contiguous: - return OP_BINARY, data - else: - return OP_BINARY, data.tobytes() else: raise TypeError("data must be bytes-like or str") @@ -290,10 +285,8 @@ def prepare_ctrl(data: Data) -> bytes: """ if isinstance(data, str): return data.encode("utf-8") - elif isinstance(data, (bytes, bytearray)): + elif isinstance(data, (bytes, bytearray, memoryview)): return bytes(data) - elif isinstance(data, memoryview): - return data.tobytes() else: raise TypeError("data must be bytes-like or str") diff --git a/src/websockets/speedups.c b/src/websockets/speedups.c index ede181e5d..fc328e528 100644 --- a/src/websockets/speedups.c +++ b/src/websockets/speedups.c @@ -13,39 +13,35 @@ static const Py_ssize_t MASK_LEN = 4; /* Similar to PyBytes_AsStringAndSize, but accepts more types */ static int -_PyBytesLike_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length) +_PyBytesLike_AsStringAndSize(PyObject *obj, PyObject **tmp, char **buffer, Py_ssize_t *length) { - // This supports bytes, bytearrays, and C-contiguous memoryview objects, - // which are the most useful data structures for handling byte streams. - // websockets.framing.prepare_data() returns only values of these types. - // Any object implementing the buffer protocol could be supported, however - // that would require allocation or copying memory, which is expensive. + // This supports bytes, bytearrays, and memoryview objects, + // which are common data structures for handling byte streams. + // websockets.framing.prepare_data() returns only these types. + // If *tmp isn't NULL, the caller gets a new reference. if (PyBytes_Check(obj)) { + *tmp = NULL; *buffer = PyBytes_AS_STRING(obj); *length = PyBytes_GET_SIZE(obj); } else if (PyByteArray_Check(obj)) { + *tmp = NULL; *buffer = PyByteArray_AS_STRING(obj); *length = PyByteArray_GET_SIZE(obj); } else if (PyMemoryView_Check(obj)) { - Py_buffer *mv_buf; - mv_buf = PyMemoryView_GET_BUFFER(obj); - if (PyBuffer_IsContiguous(mv_buf, 'C')) - { - *buffer = mv_buf->buf; - *length = mv_buf->len; - } - else + *tmp = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C'); + if (*tmp == NULL) { - PyErr_Format( - PyExc_TypeError, - "expected a contiguous memoryview"); return -1; } + Py_buffer *mv_buf; + mv_buf = PyMemoryView_GET_BUFFER(*tmp); + *buffer = mv_buf->buf; + *length = mv_buf->len; } else { @@ -74,15 +70,17 @@ apply_mask(PyObject *self, PyObject *args, PyObject *kwds) // A pointer to a char * + length will be extracted from the data and mask // arguments, possibly via a Py_buffer. + PyObject *input_tmp = NULL; char *input; Py_ssize_t input_len; + PyObject *mask_tmp = NULL; char *mask; Py_ssize_t mask_len; // Initialize a PyBytesObject then get a pointer to the underlying char * // in order to avoid an extra memory copy in PyBytes_FromStringAndSize. - PyObject *result; + PyObject *result = NULL; char *output; // Other variables. @@ -94,23 +92,23 @@ apply_mask(PyObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords( args, kwds, "OO", kwlist, &input_obj, &mask_obj)) { - return NULL; + goto exit; } - if (_PyBytesLike_AsStringAndSize(input_obj, &input, &input_len) == -1) + if (_PyBytesLike_AsStringAndSize(input_obj, &input_tmp, &input, &input_len) == -1) { - return NULL; + goto exit; } - if (_PyBytesLike_AsStringAndSize(mask_obj, &mask, &mask_len) == -1) + if (_PyBytesLike_AsStringAndSize(mask_obj, &mask_tmp, &mask, &mask_len) == -1) { - return NULL; + goto exit; } if (mask_len != MASK_LEN) { PyErr_SetString(PyExc_ValueError, "mask must contain 4 bytes"); - return NULL; + goto exit; } // Create output. @@ -118,7 +116,7 @@ apply_mask(PyObject *self, PyObject *args, PyObject *kwds) result = PyBytes_FromStringAndSize(NULL, input_len); if (result == NULL) { - return NULL; + goto exit; } // Since we juste created result, we don't need error checks. @@ -172,6 +170,9 @@ apply_mask(PyObject *self, PyObject *args, PyObject *kwds) output[i] = input[i] ^ mask[i & (MASK_LEN - 1)]; } +exit: + Py_XDECREF(input_tmp); + Py_XDECREF(mask_tmp); return result; } diff --git a/tests/legacy/test_protocol.py b/tests/legacy/test_protocol.py index 218d05376..a89bcc88b 100644 --- a/tests/legacy/test_protocol.py +++ b/tests/legacy/test_protocol.py @@ -580,10 +580,6 @@ def test_send_binary_from_memoryview(self): self.loop.run_until_complete(self.protocol.send(memoryview(b"tea"))) self.assertOneFrameSent(True, OP_BINARY, b"tea") - def test_send_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete(self.protocol.send(memoryview(b"tteeaa")[::2])) - self.assertOneFrameSent(True, OP_BINARY, b"tea") - def test_send_dict(self): with self.assertRaises(TypeError): self.loop.run_until_complete(self.protocol.send({"not": "encoded"})) @@ -624,14 +620,6 @@ def test_send_iterable_binary_from_memoryview(self): (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") ) - def test_send_iterable_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete( - self.protocol.send([memoryview(b"ttee")[::2], memoryview(b"aa")[::2]]) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - def test_send_empty_iterable(self): self.loop.run_until_complete(self.protocol.send([])) self.assertNoFrameSent() @@ -697,16 +685,6 @@ def test_send_async_iterable_binary_from_memoryview(self): (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") ) - def test_send_async_iterable_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete( - self.protocol.send( - async_iterable([memoryview(b"ttee")[::2], memoryview(b"aa")[::2]]) - ) - ) - self.assertFramesSent( - (False, OP_BINARY, b"te"), (False, OP_CONT, b"a"), (True, OP_CONT, b"") - ) - def test_send_empty_async_iterable(self): self.loop.run_until_complete(self.protocol.send(async_iterable([]))) self.assertNoFrameSent() @@ -799,10 +777,6 @@ def test_ping_binary_from_memoryview(self): self.loop.run_until_complete(self.protocol.ping(memoryview(b"tea"))) self.assertOneFrameSent(True, OP_PING, b"tea") - def test_ping_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete(self.protocol.ping(memoryview(b"tteeaa")[::2])) - self.assertOneFrameSent(True, OP_PING, b"tea") - def test_ping_type_error(self): with self.assertRaises(TypeError): self.loop.run_until_complete(self.protocol.ping(42)) @@ -856,10 +830,6 @@ def test_pong_binary_from_memoryview(self): self.loop.run_until_complete(self.protocol.pong(memoryview(b"tea"))) self.assertOneFrameSent(True, OP_PONG, b"tea") - def test_pong_binary_from_non_contiguous_memoryview(self): - self.loop.run_until_complete(self.protocol.pong(memoryview(b"tteeaa")[::2])) - self.assertOneFrameSent(True, OP_PONG, b"tea") - def test_pong_type_error(self): with self.assertRaises(TypeError): self.loop.run_until_complete(self.protocol.pong(42)) diff --git a/tests/test_frames.py b/tests/test_frames.py index 4d10c6ef2..13a712322 100644 --- a/tests/test_frames.py +++ b/tests/test_frames.py @@ -218,12 +218,6 @@ def test_prepare_data_memoryview(self): (OP_BINARY, memoryview(b"tea")), ) - def test_prepare_data_non_contiguous_memoryview(self): - self.assertEqual( - prepare_data(memoryview(b"tteeaa")[::2]), - (OP_BINARY, b"tea"), - ) - def test_prepare_data_list(self): with self.assertRaises(TypeError): prepare_data([]) @@ -246,9 +240,6 @@ def test_prepare_ctrl_bytearray(self): def test_prepare_ctrl_memoryview(self): self.assertEqual(prepare_ctrl(memoryview(b"tea")), b"tea") - def test_prepare_ctrl_non_contiguous_memoryview(self): - self.assertEqual(prepare_ctrl(memoryview(b"tteeaa")[::2]), b"tea") - def test_prepare_ctrl_list(self): with self.assertRaises(TypeError): prepare_ctrl([]) diff --git a/tests/test_utils.py b/tests/test_utils.py index b490c2409..a9ea8dcbd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -43,21 +43,18 @@ def test_apply_mask(self): self.assertEqual(result, data_out) def test_apply_mask_memoryview(self): - for data_type, mask_type in self.apply_mask_type_combos: + for mask_type in [bytes, bytearray]: for data_in, mask, data_out in self.apply_mask_test_values: - data_in, mask = data_type(data_in), mask_type(mask) - data_in, mask = memoryview(data_in), memoryview(mask) + data_in, mask = memoryview(data_in), mask_type(mask) with self.subTest(data_in=data_in, mask=mask): result = self.apply_mask(data_in, mask) self.assertEqual(result, data_out) def test_apply_mask_non_contiguous_memoryview(self): - for data_type, mask_type in self.apply_mask_type_combos: + for mask_type in [bytes, bytearray]: for data_in, mask, data_out in self.apply_mask_test_values: - data_in, mask = data_type(data_in), mask_type(mask) - data_in, mask = memoryview(data_in), memoryview(mask) - data_in, mask = data_in[::-1], mask[::-1] + data_in, mask = memoryview(data_in)[::-1], mask_type(mask)[::-1] data_out = data_out[::-1] with self.subTest(data_in=data_in, mask=mask): @@ -92,16 +89,3 @@ class SpeedupsTests(ApplyMaskTests): @staticmethod def apply_mask(*args, **kwargs): return c_apply_mask(*args, **kwargs) - - def test_apply_mask_non_contiguous_memoryview(self): - for data_type, mask_type in self.apply_mask_type_combos: - for data_in, mask, data_out in self.apply_mask_test_values: - data_in, mask = data_type(data_in), mask_type(mask) - data_in, mask = memoryview(data_in), memoryview(mask) - data_in, mask = data_in[::-1], mask[::-1] - data_out = data_out[::-1] - - with self.subTest(data_in=data_in, mask=mask): - # The C extension only supports contiguous memoryviews. - with self.assertRaises(TypeError): - self.apply_mask(data_in, mask) From 6167b5d8d8f7ec7d96f925089813503ee53b2983 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 11 Dec 2020 22:02:12 +0100 Subject: [PATCH 068/104] Clarify there's no guarantee to yield control. Fix #865. --- src/websockets/legacy/protocol.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/websockets/legacy/protocol.py b/src/websockets/legacy/protocol.py index e4592b8a0..aa1b156c6 100644 --- a/src/websockets/legacy/protocol.py +++ b/src/websockets/legacy/protocol.py @@ -555,15 +555,15 @@ async def send( its :meth:`~dict.keys` method and pass the result to :meth:`send`. Canceling :meth:`send` is discouraged. Instead, you should close the - connection with :meth:`close`. Indeed, there only two situations where - :meth:`send` yields control to the event loop: + connection with :meth:`close`. Indeed, there are only two situations + where :meth:`send` may yield control to the event loop: 1. The write buffer is full. If you don't want to wait until enough data is sent, your only alternative is to close the connection. :meth:`close` will likely time out then abort the TCP connection. - 2. ``message`` is an asynchronous iterator. Stopping in the middle of - a fragmented message will cause a protocol error. Closing the - connection has the same effect. + 2. ``message`` is an asynchronous iterator that yields control. + Stopping in the middle of a fragmented message will cause a + protocol error. Closing the connection has the same effect. :raises TypeError: for unsupported inputs From dccba0efb3bcb554fad85d72b4f6aa392626caac Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 13 Dec 2020 10:57:15 +0100 Subject: [PATCH 069/104] Fix sending fragmented, compressed messages. Fix #866. --- docs/changelog.rst | 2 ++ .../extensions/permessage_deflate.py | 26 +++++++++++-------- tests/extensions/test_permessage_deflate.py | 6 ++--- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8d255fdfd..e8a41b53c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -55,6 +55,8 @@ fixing regressions shortly after a release. * Raised an error when passing a :class:`dict` to :meth:`~legacy.protocol.WebSocketCommonProtocol.send`. +* Fixed sending fragmented, compressed messages. + * Fixed ``Host`` header sent when connecting to an IPv6 address. * Aligned maximum cookie size with popular web browsers. diff --git a/src/websockets/extensions/permessage_deflate.py b/src/websockets/extensions/permessage_deflate.py index 9a3fc4ba5..4f520af38 100644 --- a/src/websockets/extensions/permessage_deflate.py +++ b/src/websockets/extensions/permessage_deflate.py @@ -100,7 +100,7 @@ def decode(self, frame: Frame, *, max_size: Optional[int] = None) -> Frame: return frame # Handle continuation data frames: - # - skip if the initial data frame wasn't encoded + # - skip if the message isn't encoded # - reset "decode continuation data" flag if it's a final frame if frame.opcode == OP_CONT: if not self.decode_cont_data: @@ -109,21 +109,23 @@ def decode(self, frame: Frame, *, max_size: Optional[int] = None) -> Frame: self.decode_cont_data = False # Handle text and binary data frames: - # - skip if the frame isn't encoded + # - skip if the message isn't encoded + # - unset the rsv1 flag on the first frame of a compressed message # - set "decode continuation data" flag if it's a non-final frame else: if not frame.rsv1: return frame - if not frame.fin: # frame.rsv1 is True at this point + frame = frame._replace(rsv1=False) + if not frame.fin: self.decode_cont_data = True # Re-initialize per-message decoder. if self.remote_no_context_takeover: self.decoder = zlib.decompressobj(wbits=-self.remote_max_window_bits) - # Uncompress compressed frames. Protect against zip bombs by - # preventing zlib from decompressing more than max_length bytes - # (except when the limit is disabled with max_size = None). + # Uncompress data. Protect against zip bombs by preventing zlib from + # decompressing more than max_length bytes (except when the limit is + # disabled with max_size = None). data = frame.data if frame.fin: data += _EMPTY_UNCOMPRESSED_BLOCK @@ -136,7 +138,7 @@ def decode(self, frame: Frame, *, max_size: Optional[int] = None) -> Frame: if frame.fin and self.remote_no_context_takeover: del self.decoder - return frame._replace(data=data, rsv1=False) + return frame._replace(data=data) def encode(self, frame: Frame) -> Frame: """ @@ -147,17 +149,19 @@ def encode(self, frame: Frame) -> Frame: if frame.opcode in CTRL_OPCODES: return frame - # Since we always encode and never fragment messages, there's no logic - # similar to decode() here at this time. + # Since we always encode messages, there's no "encode continuation + # data" flag similar to "decode continuation data" at this time. if frame.opcode != OP_CONT: + # Set the rsv1 flag on the first frame of a compressed message. + frame = frame._replace(rsv1=True) # Re-initialize per-message decoder. if self.local_no_context_takeover: self.encoder = zlib.compressobj( wbits=-self.local_max_window_bits, **self.compress_settings ) - # Compress data frames. + # Compress data. data = self.encoder.compress(frame.data) + self.encoder.flush(zlib.Z_SYNC_FLUSH) if frame.fin and data.endswith(_EMPTY_UNCOMPRESSED_BLOCK): data = data[:-4] @@ -166,7 +170,7 @@ def encode(self, frame: Frame) -> Frame: if frame.fin and self.local_no_context_takeover: del self.encoder - return frame._replace(data=data, rsv1=True) + return frame._replace(data=data) def _build_parameters( diff --git a/tests/extensions/test_permessage_deflate.py b/tests/extensions/test_permessage_deflate.py index 328861e58..7fc4c1c3a 100644 --- a/tests/extensions/test_permessage_deflate.py +++ b/tests/extensions/test_permessage_deflate.py @@ -113,10 +113,10 @@ def test_encode_decode_fragmented_text_frame(self): frame1._replace(rsv1=True, data=b"JNL;\xbc\x12\x00\x00\x00\xff\xff"), ) self.assertEqual( - enc_frame2, frame2._replace(rsv1=True, data=b"RPS\x00\x00\x00\x00\xff\xff") + enc_frame2, frame2._replace(data=b"RPS\x00\x00\x00\x00\xff\xff") ) self.assertEqual( - enc_frame3, frame3._replace(rsv1=True, data=b"J.\xca\xcf,.N\xcc+)\x06\x00") + enc_frame3, frame3._replace(data=b"J.\xca\xcf,.N\xcc+)\x06\x00") ) dec_frame1 = self.extension.decode(enc_frame1) @@ -138,7 +138,7 @@ def test_encode_decode_fragmented_binary_frame(self): enc_frame1, frame1._replace(rsv1=True, data=b"*IMT\x00\x00\x00\x00\xff\xff") ) self.assertEqual( - enc_frame2, frame2._replace(rsv1=True, data=b"*\xc9\xccM\x05\x00") + enc_frame2, frame2._replace(data=b"*\xc9\xccM\x05\x00") ) dec_frame1 = self.extension.decode(enc_frame1) From 97a601454e193d1f30d3069d8015d086a5b83aa2 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 1 Jan 2021 18:21:24 +0100 Subject: [PATCH 070/104] Support serve() with existing Unix socket. Fix #878. --- docs/changelog.rst | 2 ++ src/websockets/legacy/server.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e8a41b53c..de4483b17 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -59,6 +59,8 @@ fixing regressions shortly after a release. * Fixed ``Host`` header sent when connecting to an IPv6 address. +* Fixed starting a Unix server listening on an existing socket. + * Aligned maximum cookie size with popular web browsers. * Ensured cancellation always propagates, even on Python versions where diff --git a/src/websockets/legacy/server.py b/src/websockets/legacy/server.py index 4dea9459d..42e0d6cf0 100644 --- a/src/websockets/legacy/server.py +++ b/src/websockets/legacy/server.py @@ -875,6 +875,7 @@ def __init__( select_subprotocol: Optional[ Callable[[Sequence[Subprotocol], Sequence[Subprotocol]], Subprotocol] ] = None, + unix: bool = False, **kwargs: Any, ) -> None: # Backwards compatibility: close_timeout used to be called timeout. @@ -931,16 +932,16 @@ def __init__( select_subprotocol=select_subprotocol, ) - if path is None: - create_server = functools.partial( - loop.create_server, factory, host, port, **kwargs - ) - else: + if unix: # unix_serve(path) must not specify host and port parameters. assert host is None and port is None create_server = functools.partial( loop.create_unix_server, factory, path, **kwargs ) + else: + create_server = functools.partial( + loop.create_server, factory, host, port, **kwargs + ) # This is a coroutine function. self._create_server = create_server @@ -981,7 +982,7 @@ async def __await_impl__(self) -> WebSocketServer: def unix_serve( ws_handler: Callable[[WebSocketServerProtocol, str], Awaitable[Any]], - path: str, + path: Optional[str] = None, **kwargs: Any, ) -> Serve: """ @@ -997,4 +998,4 @@ def unix_serve( :param path: file system path to the Unix socket """ - return serve(ws_handler, path=path, **kwargs) + return serve(ws_handler, path=path, unix=True, **kwargs) From aa93c4ceca90a1798f86b2fc2b110a42f308d721 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 1 Jan 2021 18:34:06 +0100 Subject: [PATCH 071/104] Make black happy. --- tests/extensions/test_permessage_deflate.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/extensions/test_permessage_deflate.py b/tests/extensions/test_permessage_deflate.py index 7fc4c1c3a..908cd91a4 100644 --- a/tests/extensions/test_permessage_deflate.py +++ b/tests/extensions/test_permessage_deflate.py @@ -135,10 +135,12 @@ def test_encode_decode_fragmented_binary_frame(self): enc_frame2 = self.extension.encode(frame2) self.assertEqual( - enc_frame1, frame1._replace(rsv1=True, data=b"*IMT\x00\x00\x00\x00\xff\xff") + enc_frame1, + frame1._replace(rsv1=True, data=b"*IMT\x00\x00\x00\x00\xff\xff"), ) self.assertEqual( - enc_frame2, frame2._replace(data=b"*\xc9\xccM\x05\x00") + enc_frame2, + frame2._replace(data=b"*\xc9\xccM\x05\x00"), ) dec_frame1 = self.extension.decode(enc_frame1) From dda3dfa992ddf6045be48c34143e4c1656dff9d4 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 20 Apr 2021 19:31:39 +0200 Subject: [PATCH 072/104] Document how to run on Heroku. Fix #929. --- docs/heroku.rst | 153 +++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/spelling_wordlist.txt | 2 + 3 files changed, 156 insertions(+) create mode 100644 docs/heroku.rst diff --git a/docs/heroku.rst b/docs/heroku.rst new file mode 100644 index 000000000..31c4b3f19 --- /dev/null +++ b/docs/heroku.rst @@ -0,0 +1,153 @@ +Deploying to Heroku +=================== + +This guide describes how to deploy a websockets server to Heroku_. We're going +to deploy a very simple app. The process would be identical for a more +realistic app. + +.. _Heroku: https://www.heroku.com/ + +Create application +------------------ + +Deploying to Heroku requires a git repository. Let's initialize one: + +.. code:: console + + $ mkdir websockets-echo + $ cd websockets-echo + $ git init . + Initialized empty Git repository in websockets-echo/.git/ + $ git commit --allow-empty -m "Initial commit." + [master (root-commit) 1e7947d] Initial commit. + +Follow the `set-up instructions`_ to install the Heroku CLI and to log in, if +you haven't done that yet. + +.. _set-up instructions: https://devcenter.heroku.com/articles/getting-started-with-python#set-up + +Then, create a Heroku app — if you follow these instructions step-by-step, +you'll have to pick a different name because I'm already using +``websockets-echo`` on Heroku: + +.. code:: console + + $ $ heroku create websockets-echo + Creating ⬢ websockets-echo... done + https://websockets-echo.herokuapp.com/ | https://git.heroku.com/websockets-echo.git + +Here's the implementation of the app, an echo server. Save it in a file called +``app.py``: + +.. code:: python + + #!/usr/bin/env python + + import asyncio + import os + + import websockets + + async def echo(websocket, path): + async for message in websocket: + await websocket.send(message) + + start_server = websockets.serve(echo, "", int(os.environ["PORT"])) + + asyncio.get_event_loop().run_until_complete(start_server) + asyncio.get_event_loop().run_forever() + +The server relies on the ``$PORT`` environment variable to tell on which port +it will listen, according to Heroku's conventions. + +Configure deployment +-------------------- + +In order to build the app, Heroku needs to know that it depends on websockets. +Create a ``requirements.txt`` file containing this line: + +.. code:: + + websockets + +Heroku also needs to know how to run the app. Create a ``Procfile`` with this +content: + +.. code:: + + web: python app.py + +Confirm that you created the correct files and commit them to git: + +.. code:: console + + $ ls + Procfile app.py requirements.txt + $ git add . + $ git commit -m "Deploy echo server to Heroku." + [master 8418c62] Deploy echo server to Heroku. +  3 files changed, 19 insertions(+) +  create mode 100644 Procfile +  create mode 100644 app.py +  create mode 100644 requirements.txt + +Deploy +------ + +Our app is ready. Let's deploy it! + +.. code:: console + + $ git push heroku master + + ... lots of output... + + remote: -----> Launching... + remote: Released v3 + remote: https://websockets-echo.herokuapp.com/ deployed to Heroku + remote: + remote: Verifying deploy... done. + To https://git.heroku.com/websockets-echo.git +  * [new branch] master -> master + +Validate deployment +------------------- + +Of course we'd like to confirm that our application is running as expected! + +Since it's a WebSocket server, we need a WebSocket client, such as the +interactive client that comes with websockets. + +If you're currently building a websockets server, perhaps you're already in a +virtualenv where websockets is installed. If not, you can install it in a new +virtualenv as follows: + +.. code:: console + + $ python -m venv websockets-client + $ . websockets-client/bin/activate + $ pip install websockets + +Connect the interactive client — using the name of your Heroku app instead of +``websockets-echo``: + +.. code:: console + + $ python -m websockets wss://websockets-echo.herokuapp.com/ + Connected to wss://websockets-echo.herokuapp.com/. + > + +Great! Our app is running! + +In this example, I used a secure connection (``wss://``). It worked because +Heroku served a valid TLS certificate for ``websockets-echo.herokuapp.com``. +An insecure connection (``ws://``) would also work. + +Once you're connected, you can send any message and the server will echo it, +then press Ctrl-D to terminate the connection: + +.. code:: console + + > Hello! + < Hello! + Connection closed: code = 1000 (OK), no reason. diff --git a/docs/index.rst b/docs/index.rst index 1b2f85f0a..90262ba9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -62,6 +62,7 @@ These guides will help you build and deploy a ``websockets`` application. cheatsheet deployment extensions + heroku Reference --------- diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index dd3500b73..5e0a254c7 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -22,6 +22,7 @@ cryptocurrency Ctrl daemonize fractalideas +IPv iterable keepalive KiB @@ -48,6 +49,7 @@ Unparse unregister uple username +virtualenv websocket WebSocket websockets From 93f78884ffcaf71a60d4ad20eabb603224453fa2 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 20 Apr 2021 19:37:01 +0200 Subject: [PATCH 073/104] Bump year. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index b2962adba..119b29ef3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013-2019 Aymeric Augustin and contributors. +Copyright (c) 2013-2021 Aymeric Augustin and contributors. All rights reserved. Redistribution and use in source and binary forms, with or without From 6b9e821183f8b42984e49313a7a3f5ccdd6fa8fc Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Thu, 22 Apr 2021 09:02:14 +0200 Subject: [PATCH 074/104] Clarify backwards-compatibility policy. --- docs/changelog.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index de4483b17..f3bc3a297 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,7 @@ Backwards-compatibility policy ``websockets`` also aims at providing the best API for WebSocket in Python. While we value stability, we value progress more. When an improvement requires -changing the API, we make the change and document it below. +changing a public API, we make the change and document it in this changelog. When possible with reasonable effort, we preserve backwards-compatibility for five years after the release that introduced the change. @@ -20,6 +20,9 @@ When a release contains backwards-incompatible API changes, the major version is increased, else the minor version is increased. Patch versions are only for fixing regressions shortly after a release. +Only documented APIs are public. Undocumented APIs are considered private. +They may change at any time. + 9.0 ... From c2c8bffcf5e8cae8a648c06e4cf64943550be216 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Thu, 22 Apr 2021 09:10:26 +0200 Subject: [PATCH 075/104] Improve explanation of ongoing refactoring. --- docs/changelog.rst | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f3bc3a297..4b1843713 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,8 @@ Changelog .. currentmodule:: websockets +.. _backwards-compatibility policy: + Backwards-compatibility policy .............................. @@ -32,22 +34,24 @@ They may change at any time. **Version 9.0 moves or deprecates several APIs.** + Aliases provide backwards compatibility for all previously public APIs. + * :class:`~datastructures.Headers` and :exc:`~datastructures.MultipleValuesError` were moved from - ``websockets.http`` to :mod:`websockets.datastructures`. - - * ``websockets.client``, ``websockets.server``, ``websockets.protocol``, - and ``websockets.auth`` were moved to :mod:`websockets.legacy.client`, - :mod:`websockets.legacy.server`, :mod:`websockets.legacy.protocol`, and - :mod:`websockets.legacy.auth` respectively. - - * ``websockets.handshake`` is deprecated. - - * ``websockets.http`` is deprecated. - - * ``websockets.framing`` is deprecated. - - Aliases provide backwards compatibility for all previously public APIs. + ``websockets.http`` to :mod:`websockets.datastructures`. If you're using + them, you should adjust the import path. + + * The ``client``, ``server``, ``protocol``, and ``auth`` modules were + moved from the ``websockets`` package to ``websockets.legacy`` + sub-package, as part of an upcoming refactoring. Despite the name, + they're still fully supported. The refactoring should be a transparent + upgrade for most uses when it's available. The legacy implementation + will be preserved according to the `backwards-compatibility policy`_. + + * The ``handshake``, ``http``, and ``framing`` modules in the + ``websockets`` package are deprecated. These modules provided low-level + APIs for reuse by other WebSocket implementations, but that never + happened and keeping these APIs public prevents improvements. * Added compatibility with Python 3.9. From ce1f4a071cc6651ff8bcf89f0919721aa9ca4574 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Thu, 22 Apr 2021 09:37:17 +0200 Subject: [PATCH 076/104] Deprecate headers and uri as well. They aren't involved in any public API any more. --- docs/changelog.rst | 9 +++++---- src/websockets/__init__.py | 4 ++-- src/websockets/headers.py | 3 --- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4b1843713..9b2fa4441 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -48,10 +48,11 @@ They may change at any time. upgrade for most uses when it's available. The legacy implementation will be preserved according to the `backwards-compatibility policy`_. - * The ``handshake``, ``http``, and ``framing`` modules in the - ``websockets`` package are deprecated. These modules provided low-level - APIs for reuse by other WebSocket implementations, but that never - happened and keeping these APIs public prevents improvements. + * The ``framing``, ``handshake``, ``headers``, ``http``, and ``uri`` + modules in the ``websockets`` package are deprecated. These modules + provided low-level APIs for reuse by other WebSocket implementations, + but that never happened. Keeping these APIs public makes it more + difficult to improve websockets for no actual benefit. * Added compatibility with Python 3.9. diff --git a/src/websockets/__init__.py b/src/websockets/__init__.py index 580a3960f..65d9fb913 100644 --- a/src/websockets/__init__.py +++ b/src/websockets/__init__.py @@ -96,11 +96,11 @@ "ExtensionHeader": ".typing", "ExtensionParameter": ".typing", "Subprotocol": ".typing", - "parse_uri": ".uri", - "WebSocketURI": ".uri", }, deprecated_aliases={ "framing": ".legacy", "handshake": ".legacy", + "parse_uri": ".uri", + "WebSocketURI": ".uri", }, ) diff --git a/src/websockets/headers.py b/src/websockets/headers.py index 256c66bb1..6779c9c04 100644 --- a/src/websockets/headers.py +++ b/src/websockets/headers.py @@ -2,9 +2,6 @@ :mod:`websockets.headers` provides parsers and serializers for HTTP headers used in WebSocket handshake messages. -These APIs cannot be imported from :mod:`websockets`. They must be imported -from :mod:`websockets.headers`. - """ import base64 From bb40530d4051dd1dbc0522e4d9e3e72cc7e25436 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Thu, 22 Apr 2021 09:40:27 +0200 Subject: [PATCH 077/104] Remove deprecated modules from API documentation. --- docs/api.rst | 40 ++++++++-------------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index c73cf59d3..2adc0dde4 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -23,11 +23,8 @@ For convenience, public APIs can be imported directly from the :mod:`websockets` package, unless noted otherwise. Anything that isn't listed in this document is a private API. -High-level ----------- - Server -...... +------ .. automodule:: websockets.legacy.server @@ -51,7 +48,7 @@ Server .. automethod:: select_subprotocol Client -...... +------ .. automodule:: websockets.legacy.client @@ -66,7 +63,7 @@ Client .. automethod:: handshake Shared -...... +------ .. automodule:: websockets.legacy.protocol @@ -88,7 +85,7 @@ Shared .. autoattribute:: closed Types -..... +----- .. automodule:: websockets.typing @@ -96,7 +93,7 @@ Types Per-Message Deflate Extension -............................. +----------------------------- .. automodule:: websockets.extensions.permessage_deflate @@ -105,7 +102,7 @@ Per-Message Deflate Extension .. autoclass:: ClientPerMessageDeflateFactory HTTP Basic Auth -............... +--------------- .. automodule:: websockets.legacy.auth @@ -116,34 +113,13 @@ HTTP Basic Auth .. automethod:: process_request Data structures -............... +--------------- .. automodule:: websockets.datastructures :members: Exceptions -.......... +---------- .. automodule:: websockets.exceptions :members: - -Low-level ---------- - -Data transfer -............. - -.. automodule:: websockets.framing - :members: - -URI parser -.......... - -.. automodule:: websockets.uri - :members: - -Utilities -......... - -.. automodule:: websockets.headers - :members: From c0002603eb39a9a85f89a0c83337ce398aeea7de Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 25 Apr 2021 21:39:57 +0200 Subject: [PATCH 078/104] Make HeadersLike a public API. Refs #845, #854. --- src/websockets/datastructures.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/websockets/datastructures.py b/src/websockets/datastructures.py index f70d92ad7..c8e17fa98 100644 --- a/src/websockets/datastructures.py +++ b/src/websockets/datastructures.py @@ -1,5 +1,5 @@ """ -This module defines a data structure for manipulating HTTP headers. +:mod:`websockets.datastructures` defines a class for manipulating HTTP headers. """ @@ -16,7 +16,7 @@ ) -__all__ = ["Headers", "MultipleValuesError"] +__all__ = ["Headers", "HeadersLike", "MultipleValuesError"] class MultipleValuesError(LookupError): @@ -63,7 +63,7 @@ class Headers(MutableMapping[str, str]): As long as no header occurs multiple times, :class:`Headers` behaves like :class:`dict`, except keys are lower-cased to provide case-insensitivity. - Two methods support support manipulating multiple values explicitly: + Two methods support manipulating multiple values explicitly: - :meth:`get_all` returns a list of all values for a header; - :meth:`raw_items` returns an iterator of ``(name, values)`` pairs. @@ -157,3 +157,9 @@ def raw_items(self) -> Iterator[Tuple[str, str]]: HeadersLike = Union[Headers, Mapping[str, str], Iterable[Tuple[str, str]]] +HeadersLike__doc__ = """Types accepted wherever :class:`Headers` is expected""" +# Remove try / except when dropping support for Python < 3.7 +try: + HeadersLike.__doc__ = HeadersLike__doc__ +except AttributeError: # pragma: no cover + pass From fa295a75fd0fcf53906d7aa0fe4fdcc8c7d81cd2 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 25 Apr 2021 21:41:39 +0200 Subject: [PATCH 079/104] Rewrite extensions guide. --- docs/deployment.rst | 2 ++ docs/extensions.rst | 79 ++++++++++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/docs/deployment.rst b/docs/deployment.rst index ed025094d..2331af936 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -66,6 +66,8 @@ Memory usage of a single connection is the sum of: Baseline ........ +.. _compression-settings: + Compression settings are the main factor affecting the baseline amount of memory used by each connection. diff --git a/docs/extensions.rst b/docs/extensions.rst index dea91219e..151a7e297 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -1,12 +1,12 @@ Extensions ========== -.. currentmodule:: websockets +.. currentmodule:: websockets.extensions The WebSocket protocol supports extensions_. -At the time of writing, there's only one `registered extension`_, WebSocket -Per-Message Deflate, specified in :rfc:`7692`. +At the time of writing, there's only one `registered extension`_ with a public +specification, WebSocket Per-Message Deflate, specified in :rfc:`7692`. .. _extensions: https://tools.ietf.org/html/rfc6455#section-9 .. _registered extension: https://www.iana.org/assignments/websocket/websocket.xhtml#extension-name @@ -14,24 +14,31 @@ Per-Message Deflate, specified in :rfc:`7692`. Per-Message Deflate ------------------- -:func:`~legacy.server.serve()` and :func:`~legacy.client.connect` enable the -Per-Message Deflate extension by default. You can disable this with -``compression=None``. +:func:`~websockets.legacy.client.connect` and +:func:`~websockets.legacy.server.serve` enable the Per-Message Deflate +extension by default. + +If you want to disable it, set ``compression=None``:: + + import websockets + + websockets.connect(..., compression=None) + + websockets.serve(..., compression=None) -You can also configure the Per-Message Deflate extension explicitly if you -want to customize its parameters. .. _per-message-deflate-configuration-example: -Here's an example on the server side:: +You can also configure the Per-Message Deflate extension explicitly if you +want to customize compression settings:: import websockets from websockets.extensions import permessage_deflate - websockets.serve( + websockets.connect( ..., extensions=[ - permessage_deflate.ServerPerMessageDeflateFactory( + permessage_deflate.ClientPerMessageDeflateFactory( server_max_window_bits=11, client_max_window_bits=11, compress_settings={'memLevel': 4}, @@ -39,15 +46,10 @@ Here's an example on the server side:: ], ) -Here's an example on the client side:: - - import websockets - from websockets.extensions import permessage_deflate - - websockets.connect( + websockets.serve( ..., extensions=[ - permessage_deflate.ClientPerMessageDeflateFactory( + permessage_deflate.ServerPerMessageDeflateFactory( server_max_window_bits=11, client_max_window_bits=11, compress_settings={'memLevel': 4}, @@ -55,34 +57,43 @@ Here's an example on the client side:: ], ) +The window bits and memory level values chosen in these examples reduce memory +usage. You can read more about :ref:`optimizing compression settings +`. + Refer to the API documentation of -:class:`~extensions.permessage_deflate.ServerPerMessageDeflateFactory` and -:class:`~extensions.permessage_deflate.ClientPerMessageDeflateFactory` for -details. +:class:`~permessage_deflate.ClientPerMessageDeflateFactory` and +:class:`~permessage_deflate.ServerPerMessageDeflateFactory` for details. Writing an extension -------------------- During the opening handshake, WebSocket clients and servers negotiate which extensions will be used with which parameters. Then each frame is processed by -extensions before it's sent and after it's received. +extensions before being sent or after being received. + +As a consequence, writing an extension requires implementing several classes: + +* Extension Factory: it negotiates parameters and instantiates the extension. -As a consequence writing an extension requires implementing several classes: + Clients and servers require separate extension factories with distinct APIs. -1. Extension Factory: it negotiates parameters and instantiates the extension. - Clients and servers require separate extension factories with distinct APIs. + Extension factories are the public API of an extension. -2. Extension: it decodes incoming frames and encodes outgoing frames. If the - extension is symmetrical, clients and servers can use the same class. +* Extension: it decodes incoming frames and encodes outgoing frames. + + If the extension is symmetrical, clients and servers can use the same + class. + + Extensions are initialized by extension factories, so they don't need to be + part of the public API of an extension. ``websockets`` provides abstract base classes for extension factories and -extensions. +extensions. See the API documentation for details on their methods: + +* :class:`~base.ClientExtensionFactory` and + :class:`~base.ServerExtensionFactory` for extension factories, -.. autoclass:: websockets.extensions.base.ServerExtensionFactory - :members: +* :class:`~base.Extension` for extensions. -.. autoclass:: websockets.extensions.base.ClientExtensionFactory - :members: -.. autoclass:: websockets.extensions.base.Extension - :members: From 835d16dfadd912766df99dac21e82c151eb1bda7 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 10:47:38 +0200 Subject: [PATCH 080/104] Add example of client shutdown. Fix #933. --- docs/deployment.rst | 2 +- docs/faq.rst | 11 +++++++++++ example/shutdown_client.py | 19 +++++++++++++++++++ example/{shutdown.py => shutdown_server.py} | 0 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100755 example/shutdown_client.py rename example/{shutdown.py => shutdown_server.py} (100%) diff --git a/docs/deployment.rst b/docs/deployment.rst index 2331af936..8baa8836c 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -34,7 +34,7 @@ On Unix systems, shutdown is usually triggered by sending a signal. Here's a full example for handling SIGTERM on Unix: -.. literalinclude:: ../example/shutdown.py +.. literalinclude:: ../example/shutdown_server.py :emphasize-lines: 13,17-19 This example is easily adapted to handle other signals. If you override the diff --git a/docs/faq.rst b/docs/faq.rst index eee14dda8..ff91105b4 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -142,6 +142,17 @@ See `issue 414`_. .. _issue 414: https://github.com/aaugustin/websockets/issues/414 +How do I stop a client that is continuously processing messages? +................................................................ + +You can close the connection. + +Here's an example that terminates cleanly when it receives SIGTERM on Unix: + +.. literalinclude:: ../example/shutdown_client.py + :emphasize-lines: 10-13 + + How do I disable TLS/SSL certificate verification? .................................................. diff --git a/example/shutdown_client.py b/example/shutdown_client.py new file mode 100755 index 000000000..f21c0f6fa --- /dev/null +++ b/example/shutdown_client.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +import asyncio +import signal +import websockets + +async def client(): + uri = "ws://localhost:8765" + async with websockets.connect(uri) as websocket: + # Close the connection when receiving SIGTERM. + loop = asyncio.get_event_loop() + loop.add_signal_handler( + signal.SIGTERM, loop.create_task, websocket.close()) + + # Process messages received on the connection. + async for message in websocket: + ... + +asyncio.get_event_loop().run_until_complete(client()) diff --git a/example/shutdown.py b/example/shutdown_server.py similarity index 100% rename from example/shutdown.py rename to example/shutdown_server.py From cf2453625a023868bfe760dc438a500e3ebcb931 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 21:45:32 +0200 Subject: [PATCH 081/104] Clean up signature of Protocol classes. --- src/websockets/legacy/client.py | 10 ++++++---- src/websockets/legacy/server.py | 13 +++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/websockets/legacy/client.py b/src/websockets/legacy/client.py index 27f6e8209..1c0ecf62f 100644 --- a/src/websockets/legacy/client.py +++ b/src/websockets/legacy/client.py @@ -374,7 +374,6 @@ def __init__( self, uri: str, *, - path: Optional[str] = None, create_protocol: Optional[Callable[[Any], WebSocketClientProtocol]] = None, ping_interval: Optional[float] = 20, ping_timeout: Optional[float] = 20, @@ -384,9 +383,6 @@ def __init__( read_limit: int = 2 ** 16, write_limit: int = 2 ** 16, loop: Optional[asyncio.AbstractEventLoop] = None, - legacy_recv: bool = False, - klass: Optional[Type[WebSocketClientProtocol]] = None, - timeout: Optional[float] = None, compression: Optional[str] = "deflate", origin: Optional[Origin] = None, extensions: Optional[Sequence[ClientExtensionFactory]] = None, @@ -395,6 +391,7 @@ def __init__( **kwargs: Any, ) -> None: # Backwards compatibility: close_timeout used to be called timeout. + timeout: Optional[float] = kwargs.pop("timeout", None) if timeout is None: timeout = 10 else: @@ -404,6 +401,7 @@ def __init__( close_timeout = timeout # Backwards compatibility: create_protocol used to be called klass. + klass: Optional[Type[WebSocketClientProtocol]] = kwargs.pop("klass", None) if klass is None: klass = WebSocketClientProtocol else: @@ -412,6 +410,9 @@ def __init__( if create_protocol is None: create_protocol = klass + # Backwards compatibility: recv() used to return None on closed connections + legacy_recv: bool = kwargs.pop("legacy_recv", False) + if loop is None: loop = asyncio.get_event_loop() @@ -449,6 +450,7 @@ def __init__( extra_headers=extra_headers, ) + path: Optional[str] = kwargs.pop("path", None) if path is None: host: Optional[str] port: Optional[int] diff --git a/src/websockets/legacy/server.py b/src/websockets/legacy/server.py index 42e0d6cf0..b7eed52b0 100644 --- a/src/websockets/legacy/server.py +++ b/src/websockets/legacy/server.py @@ -851,7 +851,6 @@ def __init__( host: Optional[Union[str, Sequence[str]]] = None, port: Optional[int] = None, *, - path: Optional[str] = None, create_protocol: Optional[Callable[[Any], WebSocketServerProtocol]] = None, ping_interval: Optional[float] = 20, ping_timeout: Optional[float] = 20, @@ -861,9 +860,6 @@ def __init__( read_limit: int = 2 ** 16, write_limit: int = 2 ** 16, loop: Optional[asyncio.AbstractEventLoop] = None, - legacy_recv: bool = False, - klass: Optional[Type[WebSocketServerProtocol]] = None, - timeout: Optional[float] = None, compression: Optional[str] = "deflate", origins: Optional[Sequence[Optional[Origin]]] = None, extensions: Optional[Sequence[ServerExtensionFactory]] = None, @@ -875,10 +871,10 @@ def __init__( select_subprotocol: Optional[ Callable[[Sequence[Subprotocol], Sequence[Subprotocol]], Subprotocol] ] = None, - unix: bool = False, **kwargs: Any, ) -> None: # Backwards compatibility: close_timeout used to be called timeout. + timeout: Optional[float] = kwargs.pop("timeout", None) if timeout is None: timeout = 10 else: @@ -888,6 +884,7 @@ def __init__( close_timeout = timeout # Backwards compatibility: create_protocol used to be called klass. + klass: Optional[Type[WebSocketServerProtocol]] = kwargs.pop("klass", None) if klass is None: klass = WebSocketServerProtocol else: @@ -896,6 +893,9 @@ def __init__( if create_protocol is None: create_protocol = klass + # Backwards compatibility: recv() used to return None on closed connections + legacy_recv: bool = kwargs.pop("legacy_recv", False) + if loop is None: loop = asyncio.get_event_loop() @@ -932,7 +932,8 @@ def __init__( select_subprotocol=select_subprotocol, ) - if unix: + if kwargs.pop("unix", False): + path: Optional[str] = kwargs.pop("path", None) # unix_serve(path) must not specify host and port parameters. assert host is None and port is None create_server = functools.partial( From 9c818367b2177aae6c90c3a5c4fad26e540c81bc Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 21:51:07 +0200 Subject: [PATCH 082/104] Support existing Unix sockets in unix_connect. The same fix was made for the server side, but not the client side. --- docs/changelog.rst | 2 +- src/websockets/legacy/client.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9b2fa4441..91ea23dc9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,7 +67,7 @@ They may change at any time. * Fixed ``Host`` header sent when connecting to an IPv6 address. -* Fixed starting a Unix server listening on an existing socket. +* Fixed creating a client or a server with an existing Unix socket. * Aligned maximum cookie size with popular web browsers. diff --git a/src/websockets/legacy/client.py b/src/websockets/legacy/client.py index 1c0ecf62f..219c3c9bc 100644 --- a/src/websockets/legacy/client.py +++ b/src/websockets/legacy/client.py @@ -450,8 +450,12 @@ def __init__( extra_headers=extra_headers, ) - path: Optional[str] = kwargs.pop("path", None) - if path is None: + if kwargs.pop("unix", False): + path: Optional[str] = kwargs.pop("path", None) + create_connection = functools.partial( + loop.create_unix_connection, factory, path, **kwargs + ) + else: host: Optional[str] port: Optional[int] if kwargs.get("sock") is None: @@ -465,10 +469,6 @@ def __init__( create_connection = functools.partial( loop.create_connection, factory, host, port, **kwargs ) - else: - create_connection = functools.partial( - loop.create_unix_connection, factory, path, **kwargs - ) # This is a coroutine function. self._create_connection = create_connection @@ -563,7 +563,9 @@ async def __await_impl__(self) -> WebSocketClientProtocol: connect = Connect -def unix_connect(path: str, uri: str = "ws://localhost/", **kwargs: Any) -> Connect: +def unix_connect( + path: Optional[str], uri: str = "ws://localhost/", **kwargs: Any +) -> Connect: """ Similar to :func:`connect`, but for connecting to a Unix socket. @@ -578,4 +580,4 @@ def unix_connect(path: str, uri: str = "ws://localhost/", **kwargs: Any) -> Conn :param uri: WebSocket URI """ - return connect(uri=uri, path=path, **kwargs) + return connect(uri=uri, path=path, unix=True, **kwargs) From 9223d7d72ab11824988442847d4c02d7524a61c1 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 22:15:58 +0200 Subject: [PATCH 083/104] Restore backwards-compatibility for logger names. --- src/websockets/legacy/client.py | 2 +- src/websockets/legacy/protocol.py | 2 +- src/websockets/legacy/server.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/websockets/legacy/client.py b/src/websockets/legacy/client.py index 219c3c9bc..4000375fb 100644 --- a/src/websockets/legacy/client.py +++ b/src/websockets/legacy/client.py @@ -40,7 +40,7 @@ __all__ = ["connect", "unix_connect", "WebSocketClientProtocol"] -logger = logging.getLogger(__name__) +logger = logging.getLogger("websockets.server") class WebSocketClientProtocol(WebSocketCommonProtocol): diff --git a/src/websockets/legacy/protocol.py b/src/websockets/legacy/protocol.py index aa1b156c6..84af7b626 100644 --- a/src/websockets/legacy/protocol.py +++ b/src/websockets/legacy/protocol.py @@ -60,7 +60,7 @@ __all__ = ["WebSocketCommonProtocol"] -logger = logging.getLogger(__name__) +logger = logging.getLogger("websockets.protocol") # A WebSocket connection goes through the following four states, in order: diff --git a/src/websockets/legacy/server.py b/src/websockets/legacy/server.py index b7eed52b0..8e5f97a66 100644 --- a/src/websockets/legacy/server.py +++ b/src/websockets/legacy/server.py @@ -50,7 +50,7 @@ __all__ = ["serve", "unix_serve", "WebSocketServerProtocol", "WebSocketServer"] -logger = logging.getLogger(__name__) +logger = logging.getLogger("websockets.server") HeadersLikeOrCallable = Union[HeadersLike, Callable[[str, Headers], HeadersLike]] From fcb3a4c31838b797ff609d2fdb89db7f37c527ff Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 22:32:20 +0200 Subject: [PATCH 084/104] Remove backwards-compatibility from docs after 5 years. --- src/websockets/legacy/protocol.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/websockets/legacy/protocol.py b/src/websockets/legacy/protocol.py index 84af7b626..56c4d5f6a 100644 --- a/src/websockets/legacy/protocol.py +++ b/src/websockets/legacy/protocol.py @@ -466,11 +466,6 @@ async def recv(self) -> Data: :exc:`~websockets.exceptions.ConnectionClosedError` after a protocol error or a network failure. - .. versionchanged:: 3.0 - - :meth:`recv` used to return ``None`` instead. Refer to the - changelog for details. - Canceling :meth:`recv` is safe. There's no risk of losing the next message. The next invocation of :meth:`recv` will return it. This makes it possible to enforce a timeout by wrapping :meth:`recv` in From d82a7a9de7cebec22bdcdf763ba4cb7ea75bdb76 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 21:45:51 +0200 Subject: [PATCH 085/104] Revamp API documentation. --- docs/api.rst | 125 ----------------------- docs/api/client.rst | 74 ++++++++++++++ docs/api/extensions.rst | 26 +++++ docs/api/index.rst | 50 ++++++++++ docs/api/server.rst | 105 +++++++++++++++++++ docs/api/utilities.rst | 20 ++++ docs/design.rst | 6 +- docs/index.rst | 2 +- docs/spelling_wordlist.txt | 1 + src/websockets/legacy/auth.py | 3 - src/websockets/legacy/client.py | 122 +++++++++++++++++++--- src/websockets/legacy/protocol.py | 97 +----------------- src/websockets/legacy/server.py | 161 ++++++++++++++++++++++++------ 13 files changed, 517 insertions(+), 275 deletions(-) delete mode 100644 docs/api.rst create mode 100644 docs/api/client.rst create mode 100644 docs/api/extensions.rst create mode 100644 docs/api/index.rst create mode 100644 docs/api/server.rst create mode 100644 docs/api/utilities.rst diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 2adc0dde4..000000000 --- a/docs/api.rst +++ /dev/null @@ -1,125 +0,0 @@ -API -=== - -Design ------- - -``websockets`` provides complete client and server implementations, as shown -in the :doc:`getting started guide `. These functions are built on top -of low-level APIs reflecting the two phases of the WebSocket protocol: - -1. An opening handshake, in the form of an HTTP Upgrade request; - -2. Data transfer, as framed messages, ending with a closing handshake. - -The first phase is designed to integrate with existing HTTP software. -``websockets`` provides a minimal implementation to build, parse and validate -HTTP requests and responses. - -The second phase is the core of the WebSocket protocol. ``websockets`` -provides a complete implementation on top of ``asyncio`` with a simple API. - -For convenience, public APIs can be imported directly from the -:mod:`websockets` package, unless noted otherwise. Anything that isn't listed -in this document is a private API. - -Server ------- - -.. automodule:: websockets.legacy.server - - .. autofunction:: serve(ws_handler, host=None, port=None, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origins=None, extensions=None, subprotocols=None, extra_headers=None, process_request=None, select_subprotocol=None, **kwds) - :async: - - .. autofunction:: unix_serve(ws_handler, path, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origins=None, extensions=None, subprotocols=None, extra_headers=None, process_request=None, select_subprotocol=None, **kwds) - :async: - - - .. autoclass:: WebSocketServer - - .. automethod:: close - .. automethod:: wait_closed - .. autoattribute:: sockets - - .. autoclass:: WebSocketServerProtocol(ws_handler, ws_server, *, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, origins=None, extensions=None, subprotocols=None, extra_headers=None, process_request=None, select_subprotocol=None) - - .. automethod:: handshake - .. automethod:: process_request - .. automethod:: select_subprotocol - -Client ------- - -.. automodule:: websockets.legacy.client - - .. autofunction:: connect(uri, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origin=None, extensions=None, subprotocols=None, extra_headers=None, **kwds) - :async: - - .. autofunction:: unix_connect(path, uri="ws://localhost/", *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origin=None, extensions=None, subprotocols=None, extra_headers=None, **kwds) - :async: - - .. autoclass:: WebSocketClientProtocol(*, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, origin=None, extensions=None, subprotocols=None, extra_headers=None) - - .. automethod:: handshake - -Shared ------- - -.. automodule:: websockets.legacy.protocol - - .. autoclass:: WebSocketCommonProtocol(*, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None) - - .. automethod:: close - .. automethod:: wait_closed - - .. automethod:: recv - .. automethod:: send - - .. automethod:: ping - .. automethod:: pong - - .. autoattribute:: local_address - .. autoattribute:: remote_address - - .. autoattribute:: open - .. autoattribute:: closed - -Types ------ - -.. automodule:: websockets.typing - - .. autodata:: Data - - -Per-Message Deflate Extension ------------------------------ - -.. automodule:: websockets.extensions.permessage_deflate - - .. autoclass:: ServerPerMessageDeflateFactory - - .. autoclass:: ClientPerMessageDeflateFactory - -HTTP Basic Auth ---------------- - -.. automodule:: websockets.legacy.auth - - .. autofunction:: basic_auth_protocol_factory - - .. autoclass:: BasicAuthWebSocketServerProtocol - - .. automethod:: process_request - -Data structures ---------------- - -.. automodule:: websockets.datastructures - :members: - -Exceptions ----------- - -.. automodule:: websockets.exceptions - :members: diff --git a/docs/api/client.rst b/docs/api/client.rst new file mode 100644 index 000000000..f969227a9 --- /dev/null +++ b/docs/api/client.rst @@ -0,0 +1,74 @@ +Client +====== + +.. automodule:: websockets.legacy.client + + Opening a connection + -------------------- + + .. autofunction:: connect(uri, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origin=None, extensions=None, subprotocols=None, extra_headers=None, **kwds) + :async: + + .. autofunction:: unix_connect(path, uri="ws://localhost/", *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origin=None, extensions=None, subprotocols=None, extra_headers=None, **kwds) + :async: + + Using a connection + ------------------ + + .. autoclass:: WebSocketClientProtocol(*, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, origin=None, extensions=None, subprotocols=None, extra_headers=None) + + .. autoattribute:: local_address + + .. autoattribute:: remote_address + + .. autoattribute:: open + + .. autoattribute:: closed + + .. attribute:: path + + Path of the HTTP request. + + Available once the connection is open. + + .. attribute:: request_headers + + HTTP request headers as a :class:`~websockets.http.Headers` instance. + + Available once the connection is open. + + .. attribute:: response_headers + + HTTP response headers as a :class:`~websockets.http.Headers` instance. + + Available once the connection is open. + + .. attribute:: subprotocol + + Subprotocol, if one was negotiated. + + Available once the connection is open. + + .. attribute:: close_code + + WebSocket close code. + + Available once the connection is closed. + + .. attribute:: close_reason + + WebSocket close reason. + + Available once the connection is closed. + + .. automethod:: recv + + .. automethod:: send + + .. automethod:: ping + + .. automethod:: pong + + .. automethod:: close + + .. automethod:: wait_closed diff --git a/docs/api/extensions.rst b/docs/api/extensions.rst new file mode 100644 index 000000000..635c5c426 --- /dev/null +++ b/docs/api/extensions.rst @@ -0,0 +1,26 @@ +Extensions +========== + +Per-Message Deflate +------------------- + +.. automodule:: websockets.extensions.permessage_deflate + + .. autoclass:: ClientPerMessageDeflateFactory + + .. autoclass:: ServerPerMessageDeflateFactory + +Abstract classes +---------------- + +.. automodule:: websockets.extensions.base + + .. autoclass:: Extension + :members: + + .. autoclass:: ClientExtensionFactory + :members: + + .. autoclass:: ServerExtensionFactory + :members: + diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 000000000..20bb740b3 --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,50 @@ +API +=== + +``websockets`` provides complete client and server implementations, as shown +in the :doc:`getting started guide <../intro>`. + +The process for opening and closing a WebSocket connection depends on which +side you're implementing. + +* On the client side, connecting to a server with :class:`~websockets.connect` + yields a connection object that provides methods for interacting with the + connection. Your code can open a connection, then send or receive messages. + + If you use :class:`~websockets.connect` as an asynchronous context manager, + then websockets closes the connection on exit. If not, then your code is + responsible for closing the connection. + +* On the server side, :class:`~websockets.serve` starts listening for client + connections and yields an server object that supports closing the server. + + Then, when clients connects, the server initializes a connection object and + passes it to a handler coroutine, which is where your code can send or + receive messages. This pattern is called `inversion of control`_. It's + common in frameworks implementing servers. + + When the handler coroutine terminates, websockets closes the connection. You + may also close it in the handler coroutine if you'd like. + +.. _inversion of control: https://en.wikipedia.org/wiki/Inversion_of_control + +Once the connection is open, the WebSocket protocol is symmetrical, except for +low-level details that websockets manages under the hood. The same methods are +available on client connections created with :class:`~websockets.connect` and +on server connections passed to the connection handler in the arguments. + +At this point, websockets provides the same API — and uses the same code — for +client and server connections. For convenience, common methods are documented +both in the client API and server API. + +.. toctree:: + :maxdepth: 2 + + client + server + extensions + utilities + +All public APIs can be imported from the :mod:`websockets` package, unless +noted otherwise. Anything that isn't listed in this API documentation is a +private API, with no guarantees of behavior or backwards-compatibility. diff --git a/docs/api/server.rst b/docs/api/server.rst new file mode 100644 index 000000000..16c8f6359 --- /dev/null +++ b/docs/api/server.rst @@ -0,0 +1,105 @@ +Server +====== + +.. automodule:: websockets.legacy.server + + Starting a server + ----------------- + + .. autofunction:: serve(ws_handler, host=None, port=None, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origins=None, extensions=None, subprotocols=None, extra_headers=None, process_request=None, select_subprotocol=None, **kwds) + :async: + + .. autofunction:: unix_serve(ws_handler, path, *, create_protocol=None, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, compression='deflate', origins=None, extensions=None, subprotocols=None, extra_headers=None, process_request=None, select_subprotocol=None, **kwds) + :async: + + Stopping a server + ----------------- + + .. autoclass:: WebSocketServer + + .. autoattribute:: sockets + + .. automethod:: close + .. automethod:: wait_closed + + Using a connection + ------------------ + + .. autoclass:: WebSocketServerProtocol(ws_handler, ws_server, *, ping_interval=20, ping_timeout=20, close_timeout=10, max_size=2 ** 20, max_queue=2 ** 5, read_limit=2 ** 16, write_limit=2 ** 16, loop=None, origins=None, extensions=None, subprotocols=None, extra_headers=None, process_request=None, select_subprotocol=None) + + .. autoattribute:: local_address + + .. autoattribute:: remote_address + + .. autoattribute:: open + + .. autoattribute:: closed + + .. attribute:: path + + Path of the HTTP request. + + Available once the connection is open. + + .. attribute:: request_headers + + HTTP request headers as a :class:`~websockets.http.Headers` instance. + + Available once the connection is open. + + .. attribute:: response_headers + + HTTP response headers as a :class:`~websockets.http.Headers` instance. + + Available once the connection is open. + + .. attribute:: subprotocol + + Subprotocol, if one was negotiated. + + Available once the connection is open. + + .. attribute:: close_code + + WebSocket close code. + + Available once the connection is closed. + + .. attribute:: close_reason + + WebSocket close reason. + + Available once the connection is closed. + + .. automethod:: process_request + + .. automethod:: select_subprotocol + + .. automethod:: recv + + .. automethod:: send + + .. automethod:: ping + + .. automethod:: pong + + .. automethod:: close + + .. automethod:: wait_closed + +Basic authentication +-------------------- + +.. automodule:: websockets.legacy.auth + + .. autofunction:: basic_auth_protocol_factory + + .. autoclass:: BasicAuthWebSocketServerProtocol + + .. automethod:: process_request + + .. attribute:: username + + Username of the authenticated user. + + diff --git a/docs/api/utilities.rst b/docs/api/utilities.rst new file mode 100644 index 000000000..198e928b0 --- /dev/null +++ b/docs/api/utilities.rst @@ -0,0 +1,20 @@ +Utilities +========= + +Data structures +--------------- + +.. automodule:: websockets.datastructures + :members: + +Exceptions +---------- + +.. automodule:: websockets.exceptions + :members: + +Types +----- + +.. automodule:: websockets.typing + :members: diff --git a/docs/design.rst b/docs/design.rst index f2718370d..0cabc2e5d 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -13,7 +13,7 @@ wish to understand what happens under the hood. Internals described in this document may change at any time. - Backwards compatibility is only guaranteed for `public APIs `_. + Backwards compatibility is only guaranteed for `public APIs `_. Lifecycle @@ -404,8 +404,8 @@ don't involve inversion of control. Library ....... -Most :doc:`public APIs ` of ``websockets`` are coroutines. They may be -canceled, for example if the user starts a task that calls these coroutines +Most :doc:`public APIs ` of ``websockets`` are coroutines. They may +be canceled, for example if the user starts a task that calls these coroutines and cancels the task later. ``websockets`` must handle this situation. Cancellation during the opening handshake is handled like any other exception: diff --git a/docs/index.rst b/docs/index.rst index 90262ba9a..e121fd930 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -72,7 +72,7 @@ Find all the details you could ask for, and then some. .. toctree:: :maxdepth: 2 - api + api/index Discussions ----------- diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 5e0a254c7..d7c744147 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -21,6 +21,7 @@ cryptocurrencies cryptocurrency Ctrl daemonize +datastructures fractalideas IPv iterable diff --git a/src/websockets/legacy/auth.py b/src/websockets/legacy/auth.py index 8cb60429a..e0beede57 100644 --- a/src/websockets/legacy/auth.py +++ b/src/websockets/legacy/auth.py @@ -52,9 +52,6 @@ async def process_request( """ Check HTTP Basic Auth and return a HTTP 401 or 403 response if needed. - If authentication succeeds, the username of the authenticated user is - stored in the ``username`` attribute. - """ try: authorization = request_headers["Authorization"] diff --git a/src/websockets/legacy/client.py b/src/websockets/legacy/client.py index 4000375fb..1b5bd303f 100644 --- a/src/websockets/legacy/client.py +++ b/src/websockets/legacy/client.py @@ -47,8 +47,97 @@ class WebSocketClientProtocol(WebSocketCommonProtocol): """ :class:`~asyncio.Protocol` subclass implementing a WebSocket client. - This class inherits most of its methods from - :class:`~websockets.protocol.WebSocketCommonProtocol`. + :class:`WebSocketClientProtocol`: + + * performs the opening handshake to establish the connection; + * provides :meth:`recv` and :meth:`send` coroutines for receiving and + sending messages; + * deals with control frames automatically; + * performs the closing handshake to terminate the connection. + + :class:`WebSocketClientProtocol` supports asynchronous iteration:: + + async for message in websocket: + await process(message) + + The iterator yields incoming messages. It exits normally when the + connection is closed with the close code 1000 (OK) or 1001 (going away). + It raises a :exc:`~websockets.exceptions.ConnectionClosedError` exception + when the connection is closed with any other code. + + Once the connection is open, a `Ping frame`_ is sent every + ``ping_interval`` seconds. This serves as a keepalive. It helps keeping + the connection open, especially in the presence of proxies with short + timeouts on inactive connections. Set ``ping_interval`` to ``None`` to + disable this behavior. + + .. _Ping frame: https://tools.ietf.org/html/rfc6455#section-5.5.2 + + If the corresponding `Pong frame`_ isn't received within ``ping_timeout`` + seconds, the connection is considered unusable and is closed with + code 1011. This ensures that the remote endpoint remains responsive. Set + ``ping_timeout`` to ``None`` to disable this behavior. + + .. _Pong frame: https://tools.ietf.org/html/rfc6455#section-5.5.3 + + The ``close_timeout`` parameter defines a maximum wait time for completing + the closing handshake and terminating the TCP connection. For legacy + reasons, :meth:`close` completes in at most ``5 * close_timeout`` seconds. + + ``close_timeout`` needs to be a parameter of the protocol because + websockets usually calls :meth:`close` implicitly upon exit when + :func:`connect` is used as a context manager. + + To apply a timeout to any other API, wrap it in :func:`~asyncio.wait_for`. + + The ``max_size`` parameter enforces the maximum size for incoming messages + in bytes. The default value is 1 MiB. ``None`` disables the limit. If a + message larger than the maximum size is received, :meth:`recv` will + raise :exc:`~websockets.exceptions.ConnectionClosedError` and the + connection will be closed with code 1009. + + The ``max_queue`` parameter sets the maximum length of the queue that + holds incoming messages. The default value is ``32``. ``None`` disables + the limit. Messages are added to an in-memory queue when they're received; + then :meth:`recv` pops from that queue. In order to prevent excessive + memory consumption when messages are received faster than they can be + processed, the queue must be bounded. If the queue fills up, the protocol + stops processing incoming data until :meth:`recv` is called. In this + situation, various receive buffers (at least in :mod:`asyncio` and in the + OS) will fill up, then the TCP receive window will shrink, slowing down + transmission to avoid packet loss. + + Since Python can use up to 4 bytes of memory to represent a single + character, each connection may use up to ``4 * max_size * max_queue`` + bytes of memory to store incoming messages. By default, this is 128 MiB. + You may want to lower the limits, depending on your application's + requirements. + + The ``read_limit`` argument sets the high-water limit of the buffer for + incoming bytes. The low-water limit is half the high-water limit. The + default value is 64 KiB, half of asyncio's default (based on the current + implementation of :class:`~asyncio.StreamReader`). + + The ``write_limit`` argument sets the high-water limit of the buffer for + outgoing bytes. The low-water limit is a quarter of the high-water limit. + The default value is 64 KiB, equal to asyncio's default (based on the + current implementation of ``FlowControlMixin``). + + As soon as the HTTP request and response in the opening handshake are + processed: + + * the request path is available in the :attr:`path` attribute; + * the request and response HTTP headers are available in the + :attr:`request_headers` and :attr:`response_headers` attributes, + which are :class:`~websockets.http.Headers` instances. + + If a subprotocol was negotiated, it's available in the :attr:`subprotocol` + attribute. + + Once the connection is closed, the code is available in the + :attr:`close_code` attribute and the reason in :attr:`close_reason`. + + All attributes must be treated as read-only. """ @@ -318,8 +407,12 @@ class Connect: Awaiting :func:`connect` yields a :class:`WebSocketClientProtocol` which can then be used to send and receive messages. - :func:`connect` can also be used as a asynchronous context manager. In - that case, the connection is closed when exiting the context. + :func:`connect` can also be used as a asynchronous context manager:: + + async with connect(...) as websocket: + ... + + In that case, the connection is closed when exiting the context. :func:`connect` is a wrapper around the event loop's :meth:`~asyncio.loop.create_connection` method. Unknown keyword arguments @@ -336,31 +429,28 @@ class Connect: used in the TLS handshake for secure connections and in the ``Host`` HTTP header. - The ``create_protocol`` parameter allows customizing the - :class:`~asyncio.Protocol` that manages the connection. It should be a - callable or class accepting the same arguments as - :class:`WebSocketClientProtocol` and returning an instance of - :class:`WebSocketClientProtocol` or a subclass. It defaults to - :class:`WebSocketClientProtocol`. + ``create_protocol`` defaults to :class:`WebSocketClientProtocol`. It may + be replaced by a wrapper or a subclass to customize the protocol that + manages the connection. The behavior of ``ping_interval``, ``ping_timeout``, ``close_timeout``, ``max_size``, ``max_queue``, ``read_limit``, and ``write_limit`` is - described in :class:`~websockets.protocol.WebSocketCommonProtocol`. + described in :class:`WebSocketClientProtocol`. :func:`connect` also accepts the following optional arguments: * ``compression`` is a shortcut to configure compression extensions; by default it enables the "permessage-deflate" extension; set it to - ``None`` to disable compression - * ``origin`` sets the Origin HTTP header + ``None`` to disable compression. + * ``origin`` sets the Origin HTTP header. * ``extensions`` is a list of supported extensions in order of - decreasing preference + decreasing preference. * ``subprotocols`` is a list of supported subprotocols in order of - decreasing preference + decreasing preference. * ``extra_headers`` sets additional HTTP request headers; it can be a :class:`~websockets.http.Headers` instance, a :class:`~collections.abc.Mapping`, or an iterable of ``(name, value)`` - pairs + pairs. :raises ~websockets.uri.InvalidURI: if ``uri`` is invalid :raises ~websockets.handshake.InvalidHandshake: if the opening handshake diff --git a/src/websockets/legacy/protocol.py b/src/websockets/legacy/protocol.py index 56c4d5f6a..a46e3dc4e 100644 --- a/src/websockets/legacy/protocol.py +++ b/src/websockets/legacy/protocol.py @@ -82,108 +82,13 @@ class WebSocketCommonProtocol(asyncio.Protocol): Once the WebSocket connection is established, during the data transfer phase, the protocol is almost symmetrical between the server side and the client side. :class:`WebSocketCommonProtocol` implements logic that's - shared between servers and clients.. + shared between servers and clients. Subclasses such as :class:`~websockets.legacy.server.WebSocketServerProtocol` and :class:`~websockets.legacy.client.WebSocketClientProtocol` implement the opening handshake, which is different between servers and clients. - :class:`WebSocketCommonProtocol` performs four functions: - - * It runs a task that stores incoming data frames in a queue and makes - them available with the :meth:`recv` coroutine. - * It sends outgoing data frames with the :meth:`send` coroutine. - * It deals with control frames automatically. - * It performs the closing handshake. - - :class:`WebSocketCommonProtocol` supports asynchronous iteration:: - - async for message in websocket: - await process(message) - - The iterator yields incoming messages. It exits normally when the - connection is closed with the close code 1000 (OK) or 1001 (going away). - It raises a :exc:`~websockets.exceptions.ConnectionClosedError` exception - when the connection is closed with any other code. - - Once the connection is open, a `Ping frame`_ is sent every - ``ping_interval`` seconds. This serves as a keepalive. It helps keeping - the connection open, especially in the presence of proxies with short - timeouts on inactive connections. Set ``ping_interval`` to ``None`` to - disable this behavior. - - .. _Ping frame: https://tools.ietf.org/html/rfc6455#section-5.5.2 - - If the corresponding `Pong frame`_ isn't received within ``ping_timeout`` - seconds, the connection is considered unusable and is closed with - code 1011. This ensures that the remote endpoint remains responsive. Set - ``ping_timeout`` to ``None`` to disable this behavior. - - .. _Pong frame: https://tools.ietf.org/html/rfc6455#section-5.5.3 - - The ``close_timeout`` parameter defines a maximum wait time in seconds for - completing the closing handshake and terminating the TCP connection. - :meth:`close` completes in at most ``4 * close_timeout`` on the server - side and ``5 * close_timeout`` on the client side. - - ``close_timeout`` needs to be a parameter of the protocol because - ``websockets`` usually calls :meth:`close` implicitly: - - - on the server side, when the connection handler terminates, - - on the client side, when exiting the context manager for the connection. - - To apply a timeout to any other API, wrap it in :func:`~asyncio.wait_for`. - - The ``max_size`` parameter enforces the maximum size for incoming messages - in bytes. The default value is 1 MiB. ``None`` disables the limit. If a - message larger than the maximum size is received, :meth:`recv` will - raise :exc:`~websockets.exceptions.ConnectionClosedError` and the - connection will be closed with code 1009. - - The ``max_queue`` parameter sets the maximum length of the queue that - holds incoming messages. The default value is ``32``. ``None`` disables - the limit. Messages are added to an in-memory queue when they're received; - then :meth:`recv` pops from that queue. In order to prevent excessive - memory consumption when messages are received faster than they can be - processed, the queue must be bounded. If the queue fills up, the protocol - stops processing incoming data until :meth:`recv` is called. In this - situation, various receive buffers (at least in ``asyncio`` and in the OS) - will fill up, then the TCP receive window will shrink, slowing down - transmission to avoid packet loss. - - Since Python can use up to 4 bytes of memory to represent a single - character, each connection may use up to ``4 * max_size * max_queue`` - bytes of memory to store incoming messages. By default, this is 128 MiB. - You may want to lower the limits, depending on your application's - requirements. - - The ``read_limit`` argument sets the high-water limit of the buffer for - incoming bytes. The low-water limit is half the high-water limit. The - default value is 64 KiB, half of asyncio's default (based on the current - implementation of :class:`~asyncio.StreamReader`). - - The ``write_limit`` argument sets the high-water limit of the buffer for - outgoing bytes. The low-water limit is a quarter of the high-water limit. - The default value is 64 KiB, equal to asyncio's default (based on the - current implementation of ``FlowControlMixin``). - - As soon as the HTTP request and response in the opening handshake are - processed: - - * the request path is available in the :attr:`path` attribute; - * the request and response HTTP headers are available in the - :attr:`request_headers` and :attr:`response_headers` attributes, - which are :class:`~websockets.http.Headers` instances. - - If a subprotocol was negotiated, it's available in the :attr:`subprotocol` - attribute. - - Once the connection is closed, the code is available in the - :attr:`close_code` attribute and the reason in :attr:`close_reason`. - - All these attributes must be treated as read-only. - """ # There are only two differences between the client-side and server-side diff --git a/src/websockets/legacy/server.py b/src/websockets/legacy/server.py index 8e5f97a66..e693bbd2f 100644 --- a/src/websockets/legacy/server.py +++ b/src/websockets/legacy/server.py @@ -62,11 +62,107 @@ class WebSocketServerProtocol(WebSocketCommonProtocol): """ :class:`~asyncio.Protocol` subclass implementing a WebSocket server. - This class inherits most of its methods from - :class:`~websockets.protocol.WebSocketCommonProtocol`. - - For the sake of simplicity, it doesn't rely on a full HTTP implementation. - Its support for HTTP responses is very limited. + :class:`WebSocketServerProtocol`: + + * performs the opening handshake to establish the connection; + * provides :meth:`recv` and :meth:`send` coroutines for receiving and + sending messages; + * deals with control frames automatically; + * performs the closing handshake to terminate the connection. + + You may customize the opening handshake by subclassing + :class:`WebSocketServer` and overriding: + + * :meth:`process_request` to intercept the client request before any + processing and, if appropriate, to abort the WebSocket request and + return a HTTP response instead; + * :meth:`select_subprotocol` to select a subprotocol, if the client and + the server have multiple subprotocols in common and the default logic + for choosing one isn't suitable (this is rarely needed). + + :class:`WebSocketServerProtocol` supports asynchronous iteration:: + + async for message in websocket: + await process(message) + + The iterator yields incoming messages. It exits normally when the + connection is closed with the close code 1000 (OK) or 1001 (going away). + It raises a :exc:`~websockets.exceptions.ConnectionClosedError` exception + when the connection is closed with any other code. + + Once the connection is open, a `Ping frame`_ is sent every + ``ping_interval`` seconds. This serves as a keepalive. It helps keeping + the connection open, especially in the presence of proxies with short + timeouts on inactive connections. Set ``ping_interval`` to ``None`` to + disable this behavior. + + .. _Ping frame: https://tools.ietf.org/html/rfc6455#section-5.5.2 + + If the corresponding `Pong frame`_ isn't received within ``ping_timeout`` + seconds, the connection is considered unusable and is closed with + code 1011. This ensures that the remote endpoint remains responsive. Set + ``ping_timeout`` to ``None`` to disable this behavior. + + .. _Pong frame: https://tools.ietf.org/html/rfc6455#section-5.5.3 + + The ``close_timeout`` parameter defines a maximum wait time for completing + the closing handshake and terminating the TCP connection. For legacy + reasons, :meth:`close` completes in at most ``4 * close_timeout`` seconds. + + ``close_timeout`` needs to be a parameter of the protocol because + websockets usually calls :meth:`close` implicitly when the connection + handler terminates. + + To apply a timeout to any other API, wrap it in :func:`~asyncio.wait_for`. + + The ``max_size`` parameter enforces the maximum size for incoming messages + in bytes. The default value is 1 MiB. ``None`` disables the limit. If a + message larger than the maximum size is received, :meth:`recv` will + raise :exc:`~websockets.exceptions.ConnectionClosedError` and the + connection will be closed with code 1009. + + The ``max_queue`` parameter sets the maximum length of the queue that + holds incoming messages. The default value is ``32``. ``None`` disables + the limit. Messages are added to an in-memory queue when they're received; + then :meth:`recv` pops from that queue. In order to prevent excessive + memory consumption when messages are received faster than they can be + processed, the queue must be bounded. If the queue fills up, the protocol + stops processing incoming data until :meth:`recv` is called. In this + situation, various receive buffers (at least in :mod:`asyncio` and in the + OS) will fill up, then the TCP receive window will shrink, slowing down + transmission to avoid packet loss. + + Since Python can use up to 4 bytes of memory to represent a single + character, each connection may use up to ``4 * max_size * max_queue`` + bytes of memory to store incoming messages. By default, this is 128 MiB. + You may want to lower the limits, depending on your application's + requirements. + + The ``read_limit`` argument sets the high-water limit of the buffer for + incoming bytes. The low-water limit is half the high-water limit. The + default value is 64 KiB, half of asyncio's default (based on the current + implementation of :class:`~asyncio.StreamReader`). + + The ``write_limit`` argument sets the high-water limit of the buffer for + outgoing bytes. The low-water limit is a quarter of the high-water limit. + The default value is 64 KiB, equal to asyncio's default (based on the + current implementation of ``FlowControlMixin``). + + As soon as the HTTP request and response in the opening handshake are + processed: + + * the request path is available in the :attr:`path` attribute; + * the request and response HTTP headers are available in the + :attr:`request_headers` and :attr:`response_headers` attributes, + which are :class:`~websockets.http.Headers` instances. + + If a subprotocol was negotiated, it's available in the :attr:`subprotocol` + attribute. + + Once the connection is closed, the code is available in the + :attr:`close_code` attribute and the reason in :attr:`close_reason`. + + All attributes must be treated as read-only. """ @@ -487,7 +583,7 @@ def select_subprotocol( Instead of subclassing, it is possible to override this method by passing a ``select_subprotocol`` argument to the :func:`serve` - function or the :class:`WebSocketServerProtocol` constructor + function or the :class:`WebSocketServerProtocol` constructor. :param client_subprotocols: list of subprotocols offered by the client :param server_subprotocols: list of subprotocols available on the server @@ -780,66 +876,69 @@ class Serve: :exc:`~websockets.exceptions.ConnectionClosedOK` exception on their current or next interaction with the WebSocket connection. - :func:`serve` can also be used as an asynchronous context manager. In - this case, the server is shut down when exiting the context. + :func:`serve` can also be used as an asynchronous context manager:: + + stop = asyncio.Future() # set this future to exit the server + + async with serve(...): + await stop + + In this case, the server is shut down when exiting the context. :func:`serve` is a wrapper around the event loop's :meth:`~asyncio.loop.create_server` method. It creates and starts a - :class:`~asyncio.Server` with :meth:`~asyncio.loop.create_server`. Then it - wraps the :class:`~asyncio.Server` in a :class:`WebSocketServer` and + :class:`asyncio.Server` with :meth:`~asyncio.loop.create_server`. Then it + wraps the :class:`asyncio.Server` in a :class:`WebSocketServer` and returns the :class:`WebSocketServer`. - The ``ws_handler`` argument is the WebSocket handler. It must be a - coroutine accepting two arguments: a :class:`WebSocketServerProtocol` and - the request URI. + ``ws_handler`` is the WebSocket handler. It must be a coroutine accepting + two arguments: the WebSocket connection, which is an instance of + :class:`WebSocketServerProtocol`, and the path of the request. The ``host`` and ``port`` arguments, as well as unrecognized keyword - arguments, are passed along to :meth:`~asyncio.loop.create_server`. + arguments, are passed to :meth:`~asyncio.loop.create_server`. For example, you can set the ``ssl`` keyword argument to a :class:`~ssl.SSLContext` to enable TLS. - The ``create_protocol`` parameter allows customizing the - :class:`~asyncio.Protocol` that manages the connection. It should be a - callable or class accepting the same arguments as - :class:`WebSocketServerProtocol` and returning an instance of - :class:`WebSocketServerProtocol` or a subclass. It defaults to - :class:`WebSocketServerProtocol`. + ``create_protocol`` defaults to :class:`WebSocketServerProtocol`. It may + be replaced by a wrapper or a subclass to customize the protocol that + manages the connection. The behavior of ``ping_interval``, ``ping_timeout``, ``close_timeout``, ``max_size``, ``max_queue``, ``read_limit``, and ``write_limit`` is - described in :class:`~websockets.protocol.WebSocketCommonProtocol`. + described in :class:`WebSocketServerProtocol`. :func:`serve` also accepts the following optional arguments: * ``compression`` is a shortcut to configure compression extensions; by default it enables the "permessage-deflate" extension; set it to - ``None`` to disable compression - * ``origins`` defines acceptable Origin HTTP headers; include ``None`` if - the lack of an origin is acceptable + ``None`` to disable compression. + * ``origins`` defines acceptable Origin HTTP headers; include ``None`` in + the list if the lack of an origin is acceptable. * ``extensions`` is a list of supported extensions in order of - decreasing preference + decreasing preference. * ``subprotocols`` is a list of supported subprotocols in order of - decreasing preference + decreasing preference. * ``extra_headers`` sets additional HTTP response headers when the handshake succeeds; it can be a :class:`~websockets.http.Headers` instance, a :class:`~collections.abc.Mapping`, an iterable of ``(name, value)`` pairs, or a callable taking the request path and headers in - arguments and returning one of the above + arguments and returning one of the above. * ``process_request`` allows intercepting the HTTP request; it must be a coroutine taking the request path and headers in argument; see - :meth:`~WebSocketServerProtocol.process_request` for details + :meth:`~WebSocketServerProtocol.process_request` for details. * ``select_subprotocol`` allows customizing the logic for selecting a subprotocol; it must be a callable taking the subprotocols offered by the client and available on the server in argument; see - :meth:`~WebSocketServerProtocol.select_subprotocol` for details + :meth:`~WebSocketServerProtocol.select_subprotocol` for details. Since there's no useful way to propagate exceptions triggered in handlers, - they're sent to the ``'websockets.legacy.server'`` logger instead. + they're sent to the ``"websockets.server"`` logger instead. Debugging is much easier if you configure logging to print them:: import logging - logger = logging.getLogger("websockets.legacy.server") + logger = logging.getLogger("websockets.server") logger.setLevel(logging.ERROR) logger.addHandler(logging.StreamHandler()) From 927287380011e4388c11c24d286beef2b877284d Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 22:49:58 +0200 Subject: [PATCH 086/104] Work around coverage bug. --- src/websockets/legacy/protocol.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/websockets/legacy/protocol.py b/src/websockets/legacy/protocol.py index a46e3dc4e..e4c6d63c5 100644 --- a/src/websockets/legacy/protocol.py +++ b/src/websockets/legacy/protocol.py @@ -549,7 +549,9 @@ async def send( # Other fragments. # https://github.com/python/mypy/issues/5738 - async for message_chunk in aiter_message: # type: ignore + # coverage reports this code as not covered, but it is + # exercised by tests - changing it breaks the tests! + async for message_chunk in aiter_message: # type: ignore # pragma: no cover # noqa confirm_opcode, data = prepare_data(message_chunk) if confirm_opcode != opcode: raise TypeError("data contains inconsistent types") From 5ab214b00f38cae3976fce5a315fbfa30762b60d Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 22:39:12 +0200 Subject: [PATCH 087/104] Bump version number --- docs/changelog.rst | 7 ++++++- docs/conf.py | 4 ++-- src/websockets/version.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 91ea23dc9..2644d3735 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,11 +25,16 @@ fixing regressions shortly after a release. Only documented APIs are public. Undocumented APIs are considered private. They may change at any time. -9.0 +9.1 ... *In development* +9.0 +... + +*May 1, 2021* + .. note:: **Version 9.0 moves or deprecates several APIs.** diff --git a/docs/conf.py b/docs/conf.py index 0c00b96fb..dad7475f7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = '8.1' +version = '9.0' # The full version, including alpha/beta/rc tags. -release = '8.1' +release = '9.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/websockets/version.py b/src/websockets/version.py index 7377332e1..94d9f2ead 100644 --- a/src/websockets/version.py +++ b/src/websockets/version.py @@ -1 +1 @@ -version = "8.1" +version = "9.0" From 5d6fcf96cd81680e35cba00ed52cb12bf2c8f544 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 1 May 2021 22:59:03 +0200 Subject: [PATCH 088/104] Python 3.9 is now released. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7be85d7f9..8a1441209 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,7 @@ jobs: - run: tox -e py38 py39: docker: - - image: circleci/python:3.9.0b1 + - image: circleci/python:3.9 steps: # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc From 56be5f71e273fee7a2ef86166838f574b58e3c59 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 2 May 2021 15:36:56 +0200 Subject: [PATCH 089/104] Build wheels on Python 3.9. --- .appveyor.yml | 2 +- .travis.yml | 2 +- setup.cfg | 2 +- setup.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index d34b15aed..ef17ebba5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,7 +6,7 @@ skip_branch_with_pr: true environment: # websockets only works on Python >= 3.6. - CIBW_BUILD: cp36-* cp37-* cp38-* + CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* CIBW_TEST_COMMAND: python -W default -m unittest WEBSOCKETS_TESTS_TIMEOUT_FACTOR: 100 diff --git a/.travis.yml b/.travis.yml index e31c9ea0b..f2bfc724e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ env: global: # websockets only works on Python >= 3.6. - - CIBW_BUILD="cp36-* cp37-* cp38-*" + - CIBW_BUILD="cp36-* cp37-* cp38-* cp39-*"" - CIBW_TEST_COMMAND="python3 -W default -m unittest" - WEBSOCKETS_TESTS_TIMEOUT_FACTOR=100 diff --git a/setup.cfg b/setup.cfg index 5448b0f9b..04b792989 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bdist_wheel] -python-tag = py36.py37 +python-tag = py36.py37.py38.py39 [metadata] license_file = LICENSE diff --git a/setup.py b/setup.py index 85d899cb4..5adb8e835 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], package_dir = {'': 'src'}, package_data = {'websockets': ['py.typed']}, From cbae1fb00e07a880bc7e9b566249afa474469c0d Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 2 May 2021 15:36:56 +0200 Subject: [PATCH 090/104] Setup GitHub actions. --- .github/workflows/tests.yml | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..eb06ebfea --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,52 @@ +name: Run tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + main: + name: Run code quality checks + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Install Python 3.x + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install tox + run: pip install tox + - name: Run tests with coverage + run: tox -e coverage + - name: Check code formatting + run: tox -e black + - name: Check code style + run: tox -e flake8 + - name: Check imports ordering + run: tox -e isort + - name: Check types statically + run: tox -e mypy + + matrix: + name: Run tests on Python ${{ matrix.python }} + needs: main + runs-on: ubuntu-latest + strategy: + matrix: + python: [3.6, 3.7, 3.8, 3.9] + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Install Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install tox + run: pip install tox + - name: Run tests + run: tox -e py From 3d55449d5df642d6be401c21afee450edb8c4422 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 2 May 2021 17:57:53 +0200 Subject: [PATCH 091/104] Drop CircleCI setup. --- .circleci/config.yml | 67 -------------------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 8a1441209..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,67 +0,0 @@ -version: 2 - -jobs: - main: - docker: - - image: circleci/python:3.7 - steps: - # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. - - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc - - checkout - - run: sudo pip install tox codecov - - run: tox -e coverage,black,flake8,isort,mypy - - run: codecov - py36: - docker: - - image: circleci/python:3.6 - steps: - # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. - - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc - - checkout - - run: sudo pip install tox - - run: tox -e py36 - py37: - docker: - - image: circleci/python:3.7 - steps: - # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. - - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc - - checkout - - run: sudo pip install tox - - run: tox -e py37 - py38: - docker: - - image: circleci/python:3.8 - steps: - # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. - - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc - - checkout - - run: sudo pip install tox - - run: tox -e py38 - py39: - docker: - - image: circleci/python:3.9 - steps: - # Remove IPv6 entry for localhost in Circle CI containers because it doesn't work anyway. - - run: sudo cp /etc/hosts /tmp; sudo sed -i '/::1/d' /tmp/hosts; sudo cp /tmp/hosts /etc - - checkout - - run: sudo pip install tox - - run: tox -e py39 - -workflows: - version: 2 - build: - jobs: - - main - - py36: - requires: - - main - - py37: - requires: - - main - - py38: - requires: - - main - - py39: - requires: - - main From c3d7b7f6565bd2a40aa5cdd5d0e44642148d41e2 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 2 May 2021 17:57:59 +0200 Subject: [PATCH 092/104] Change badge in README. --- README.rst | 9 +++------ docs/index.rst | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 1e15ba198..bda73c640 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ :width: 480px :alt: websockets -|rtd| |pypi-v| |pypi-pyversions| |pypi-l| |pypi-wheel| |circleci| |codecov| +|rtd| |pypi-v| |pypi-pyversions| |pypi-l| |pypi-wheel| |tests| .. |rtd| image:: https://readthedocs.org/projects/websockets/badge/?version=latest :target: https://websockets.readthedocs.io/ @@ -19,11 +19,8 @@ .. |pypi-wheel| image:: https://img.shields.io/pypi/wheel/websockets.svg :target: https://pypi.python.org/pypi/websockets -.. |circleci| image:: https://img.shields.io/circleci/project/github/aaugustin/websockets.svg - :target: https://circleci.com/gh/aaugustin/websockets - -.. |codecov| image:: https://codecov.io/gh/aaugustin/websockets/branch/master/graph/badge.svg - :target: https://codecov.io/gh/aaugustin/websockets +.. |tests| image:: https://github.com/aaugustin/websockets/workflows/tests/badge.svg?branch=master + :target: https://github.com/aaugustin/websockets/actions?workflow=tests What is ``websockets``? ----------------------- diff --git a/docs/index.rst b/docs/index.rst index e121fd930..5914d7289 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ websockets ========== -|pypi-v| |pypi-pyversions| |pypi-l| |pypi-wheel| |circleci| |codecov| +|pypi-v| |pypi-pyversions| |pypi-l| |pypi-wheel| |tests| .. |pypi-v| image:: https://img.shields.io/pypi/v/websockets.svg :target: https://pypi.python.org/pypi/websockets @@ -15,11 +15,8 @@ websockets .. |pypi-wheel| image:: https://img.shields.io/pypi/wheel/websockets.svg :target: https://pypi.python.org/pypi/websockets -.. |circleci| image:: https://img.shields.io/circleci/project/github/aaugustin/websockets.svg - :target: https://circleci.com/gh/aaugustin/websockets - -.. |codecov| image:: https://codecov.io/gh/aaugustin/websockets/branch/master/graph/badge.svg - :target: https://codecov.io/gh/aaugustin/websockets +.. |tests| image:: https://github.com/aaugustin/websockets/workflows/tests/badge.svg?branch=master + :target: https://github.com/aaugustin/websockets/actions?workflow=tests ``websockets`` is a library for building WebSocket servers_ and clients_ in Python with a focus on correctness and simplicity. From a45cc5afe067925759a2644bd9ef9b5346adefa1 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 2 May 2021 20:16:16 +0200 Subject: [PATCH 093/104] Build distributions on GitHub actions. --- .github/workflows/wheels.yml | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..7ea97c61f --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,71 @@ +name: Build wheels + +on: + push: + branches: + - main + tags: + - '*' + +jobs: + sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Install Python 3.x + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Build sdist + run: python setup.py sdist + - name: Save sdist + uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, windows-2019, macOS-10.15] + + steps: + - name: Check out repository + uses: actions/checkout@v2 + - name: Make extension build mandatory + run: touch .cibuildwheel + - name: Install Python 3.x + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: all + - name: Build wheels + uses: joerick/cibuildwheel@v1.11.0 + env: + CIBW_ARCHS_LINUX: auto aarch64 + CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* + - name: Save wheels + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + + upload_pypi: + name: Upload to PyPI + needs: [sdist, wheels] + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + - uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} From b0d211d0f32633977e73f51a1573e6a07319a0b0 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 2 May 2021 20:23:38 +0200 Subject: [PATCH 094/104] Drop Travis CI and Appveyor setup. --- .appveyor.yml | 27 --------------------------- .travis.yml | 43 ------------------------------------------- 2 files changed, 70 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .travis.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index ef17ebba5..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,27 +0,0 @@ -branches: - only: - - master - -skip_branch_with_pr: true - -environment: -# websockets only works on Python >= 3.6. - CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* - CIBW_TEST_COMMAND: python -W default -m unittest - WEBSOCKETS_TESTS_TIMEOUT_FACTOR: 100 - -install: -# Ensure python is Python 3. - - set PATH=C:\Python37;%PATH% - - cmd: python -m pip install --upgrade cibuildwheel -# Create file '.cibuildwheel' so that extension build is not optional (c.f. setup.py). - - cmd: touch .cibuildwheel - -build_script: - - cmd: python -m cibuildwheel --output-dir wheelhouse -# Upload to PyPI on tags - - ps: >- - if ($env:APPVEYOR_REPO_TAG -eq "true") { - Invoke-Expression "python -m pip install twine" - Invoke-Expression "python -m twine upload --skip-existing wheelhouse/*.whl" - } diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f2bfc724e..000000000 --- a/.travis.yml +++ /dev/null @@ -1,43 +0,0 @@ -env: - global: - # websockets only works on Python >= 3.6. - - CIBW_BUILD="cp36-* cp37-* cp38-* cp39-*"" - - CIBW_TEST_COMMAND="python3 -W default -m unittest" - - WEBSOCKETS_TESTS_TIMEOUT_FACTOR=100 - -matrix: - include: - - language: python - dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) - sudo: required - python: "3.7" - services: - - docker - - language: python - dist: xenial - sudo: required - python: "3.7" - arch: arm64 - services: - - docker - - os: osx - osx_image: xcode8.3 - -install: -# Python 3 is needed to run cibuildwheel for websockets. - - if [ "${TRAVIS_OS_NAME:-}" == "osx" ]; then - brew update; - brew upgrade python; - fi -# Install cibuildwheel using pip3 to make sure Python 3 is used. - - pip3 install --upgrade cibuildwheel -# Create file '.cibuildwheel' so that extension build is not optional (c.f. setup.py). - - touch .cibuildwheel - -script: - - cibuildwheel --output-dir wheelhouse -# Upload to PyPI on tags - - if [ "${TRAVIS_TAG:-}" != "" ]; then - pip3 install twine; - python3 -m twine upload --skip-existing wheelhouse/*; - fi From fc176f462b6a5ef4f470df415780b09fed5da7c1 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 2 May 2021 20:49:05 +0200 Subject: [PATCH 095/104] Bump version number. --- docs/changelog.rst | 7 +++++++ src/websockets/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2644d3735..1e5f92211 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,6 +30,13 @@ They may change at any time. *In development* +9.0.1 +..... + +*May 2, 2021* + +* Fixed issues with the packaging of the 9.0 release. + 9.0 ... diff --git a/src/websockets/version.py b/src/websockets/version.py index 94d9f2ead..23b7f329b 100644 --- a/src/websockets/version.py +++ b/src/websockets/version.py @@ -1 +1 @@ -version = "9.0" +version = "9.0.1" From 217ac2d19174c6f01d9524648eb4058985f72754 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Thu, 13 May 2021 22:37:31 +0200 Subject: [PATCH 096/104] Fix broken link. Fix #953. --- docs/design.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design.rst b/docs/design.rst index 0cabc2e5d..61b42b528 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -13,7 +13,7 @@ wish to understand what happens under the hood. Internals described in this document may change at any time. - Backwards compatibility is only guaranteed for `public APIs `_. + Backwards compatibility is only guaranteed for :doc:`public APIs `. Lifecycle From 70fadbf97c5a117ca13f6c8f4f111ba5025f3c94 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Thu, 13 May 2021 22:41:46 +0200 Subject: [PATCH 097/104] Restore compatibility with Python < 3.9. Fix #951. --- docs/changelog.rst | 7 +++++++ src/websockets/__main__.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1e5f92211..fb40aee2a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,6 +30,13 @@ They may change at any time. *In development* +9.0.2 +..... + +*In development* + +* Restored compatibility of ``python -m websockets`` with Python < 3.9. + 9.0.1 ..... diff --git a/src/websockets/__main__.py b/src/websockets/__main__.py index d44e34e74..fb126997a 100644 --- a/src/websockets/__main__.py +++ b/src/websockets/__main__.py @@ -178,11 +178,11 @@ def main() -> None: # Due to zealous removal of the loop parameter in the Queue constructor, # we need a factory coroutine to run in the freshly created event loop. - async def queue_factory() -> asyncio.Queue[str]: + async def queue_factory() -> "asyncio.Queue[str]": return asyncio.Queue() # Create a queue of user inputs. There's no need to limit its size. - inputs: asyncio.Queue[str] = loop.run_until_complete(queue_factory()) + inputs: "asyncio.Queue[str]" = loop.run_until_complete(queue_factory()) # Create a stop condition when receiving SIGINT or SIGTERM. stop: asyncio.Future[None] = loop.create_future() From e44e085e030d186c7bb9822becfbb5423aefe971 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 7 May 2021 21:21:37 +0200 Subject: [PATCH 098/104] Use relative imports everywhere, for consistency. Fix #946. --- docs/api/extensions.rst | 2 +- docs/extensions.rst | 7 +++---- src/websockets/extensions/__init__.py | 4 ++++ src/websockets/frames.py | 6 +++--- src/websockets/legacy/framing.py | 6 +++--- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/api/extensions.rst b/docs/api/extensions.rst index 635c5c426..71f015bb2 100644 --- a/docs/api/extensions.rst +++ b/docs/api/extensions.rst @@ -13,7 +13,7 @@ Per-Message Deflate Abstract classes ---------------- -.. automodule:: websockets.extensions.base +.. automodule:: websockets.extensions .. autoclass:: Extension :members: diff --git a/docs/extensions.rst b/docs/extensions.rst index 151a7e297..042ed3d9a 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -91,9 +91,8 @@ As a consequence, writing an extension requires implementing several classes: ``websockets`` provides abstract base classes for extension factories and extensions. See the API documentation for details on their methods: -* :class:`~base.ClientExtensionFactory` and - :class:`~base.ServerExtensionFactory` for extension factories, - -* :class:`~base.Extension` for extensions. +* :class:`ClientExtensionFactory` and class:`ServerExtensionFactory` for + :extension factories, +* :class:`Extension` for extensions. diff --git a/src/websockets/extensions/__init__.py b/src/websockets/extensions/__init__.py index e69de29bb..02838b98a 100644 --- a/src/websockets/extensions/__init__.py +++ b/src/websockets/extensions/__init__.py @@ -0,0 +1,4 @@ +from .base import * + + +__all__ = ["Extension", "ClientExtensionFactory", "ServerExtensionFactory"] diff --git a/src/websockets/frames.py b/src/websockets/frames.py index 71783e176..6e5ef1b73 100644 --- a/src/websockets/frames.py +++ b/src/websockets/frames.py @@ -103,7 +103,7 @@ def parse( *, mask: bool, max_size: Optional[int] = None, - extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + extensions: Optional[Sequence["extensions.Extension"]] = None, ) -> Generator[None, None, "Frame"]: """ Read a WebSocket frame. @@ -172,7 +172,7 @@ def serialize( self, *, mask: bool, - extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + extensions: Optional[Sequence["extensions.Extension"]] = None, ) -> bytes: """ Write a WebSocket frame. @@ -338,4 +338,4 @@ def check_close(code: int) -> None: # at the bottom to allow circular import, because Extension depends on Frame -import websockets.extensions.base # isort:skip # noqa +from . import extensions # isort:skip # noqa diff --git a/src/websockets/legacy/framing.py b/src/websockets/legacy/framing.py index e41c295dd..627e6922c 100644 --- a/src/websockets/legacy/framing.py +++ b/src/websockets/legacy/framing.py @@ -31,7 +31,7 @@ async def read( *, mask: bool, max_size: Optional[int] = None, - extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + extensions: Optional[Sequence["extensions.Extension"]] = None, ) -> "Frame": """ Read a WebSocket frame. @@ -102,7 +102,7 @@ def write( write: Callable[[bytes], Any], *, mask: bool, - extensions: Optional[Sequence["websockets.extensions.base.Extension"]] = None, + extensions: Optional[Sequence["extensions.Extension"]] = None, ) -> None: """ Write a WebSocket frame. @@ -132,4 +132,4 @@ def write( # at the bottom to allow circular import, because Extension depends on Frame -import websockets.extensions.base # isort:skip # noqa +from .. import extensions # isort:skip # noqa From b99c4fe390a22cc846ce550a29f2c9841e99660d Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 7 May 2021 21:44:51 +0200 Subject: [PATCH 099/104] Restore real imports for compatibility with mypy. Fix #940. --- docs/api/client.rst | 2 +- docs/api/index.rst | 9 +++++++-- docs/api/server.rst | 4 ++-- docs/changelog.rst | 16 ++++++++++++++++ docs/extensions.rst | 5 ++--- src/websockets/auth.py | 2 ++ src/websockets/client.py | 12 +++--------- src/websockets/server.py | 12 ++---------- 8 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 src/websockets/auth.py diff --git a/docs/api/client.rst b/docs/api/client.rst index f969227a9..db8cbc914 100644 --- a/docs/api/client.rst +++ b/docs/api/client.rst @@ -1,7 +1,7 @@ Client ====== -.. automodule:: websockets.legacy.client +.. automodule:: websockets.client Opening a connection -------------------- diff --git a/docs/api/index.rst b/docs/api/index.rst index 20bb740b3..0a616cbce 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -46,5 +46,10 @@ both in the client API and server API. utilities All public APIs can be imported from the :mod:`websockets` package, unless -noted otherwise. Anything that isn't listed in this API documentation is a -private API, with no guarantees of behavior or backwards-compatibility. +noted otherwise. This convenience feature is incompatible with static code +analysis tools such as mypy_, though. + +.. _mypy: https://github.com/python/mypy + +Anything that isn't listed in this API documentation is a private API. There's +no guarantees of behavior or backwards-compatibility for private APIs. diff --git a/docs/api/server.rst b/docs/api/server.rst index 16c8f6359..9e7b801a9 100644 --- a/docs/api/server.rst +++ b/docs/api/server.rst @@ -1,7 +1,7 @@ Server ====== -.. automodule:: websockets.legacy.server +.. automodule:: websockets.server Starting a server ----------------- @@ -90,7 +90,7 @@ Server Basic authentication -------------------- -.. automodule:: websockets.legacy.auth +.. automodule:: websockets.auth .. autofunction:: basic_auth_protocol_factory diff --git a/docs/changelog.rst b/docs/changelog.rst index fb40aee2a..218bbec3d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,8 @@ They may change at any time. * Restored compatibility of ``python -m websockets`` with Python < 3.9. +* Restored compatibility with mypy. + 9.0.1 ..... @@ -73,6 +75,20 @@ They may change at any time. but that never happened. Keeping these APIs public makes it more difficult to improve websockets for no actual benefit. +.. note:: + + **Version 9.0 may require changes if you use static code analysis tools.** + + Convenience imports from the ``websockets`` module are performed lazily. + While this is supported by Python, static code analysis tools such as mypy + are unable to understand the behavior. + + If you depend on such tools, use the real import path, which can be found + in the API documentation:: + + from websockets.client import connect + from websockets.server import serve + * Added compatibility with Python 3.9. * Added support for IRIs in addition to URIs. diff --git a/docs/extensions.rst b/docs/extensions.rst index 042ed3d9a..f5e2f497f 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -14,9 +14,8 @@ specification, WebSocket Per-Message Deflate, specified in :rfc:`7692`. Per-Message Deflate ------------------- -:func:`~websockets.legacy.client.connect` and -:func:`~websockets.legacy.server.serve` enable the Per-Message Deflate -extension by default. +:func:`~websockets.client.connect` and :func:`~websockets.server.serve` enable +the Per-Message Deflate extension by default. If you want to disable it, set ``compression=None``:: diff --git a/src/websockets/auth.py b/src/websockets/auth.py new file mode 100644 index 000000000..f97c1feb0 --- /dev/null +++ b/src/websockets/auth.py @@ -0,0 +1,2 @@ +# See #940 for why lazy_import isn't used here for backwards compatibility. +from .legacy.auth import * # noqa diff --git a/src/websockets/client.py b/src/websockets/client.py index 91dd1662e..0ddf19f00 100644 --- a/src/websockets/client.py +++ b/src/websockets/client.py @@ -24,7 +24,6 @@ ) from .http import USER_AGENT, build_host from .http11 import Request, Response -from .imports import lazy_import from .typing import ( ConnectionOption, ExtensionHeader, @@ -36,14 +35,9 @@ from .utils import accept_key, generate_key -lazy_import( - globals(), - aliases={ - "connect": ".legacy.client", - "unix_connect": ".legacy.client", - "WebSocketClientProtocol": ".legacy.client", - }, -) +# See #940 for why lazy_import isn't used here for backwards compatibility. +from .legacy.client import * # isort:skip # noqa + __all__ = ["ClientConnection"] diff --git a/src/websockets/server.py b/src/websockets/server.py index 67ab83031..f57d36b70 100644 --- a/src/websockets/server.py +++ b/src/websockets/server.py @@ -26,7 +26,6 @@ ) from .http import USER_AGENT from .http11 import Request, Response -from .imports import lazy_import from .typing import ( ConnectionOption, ExtensionHeader, @@ -37,15 +36,8 @@ from .utils import accept_key -lazy_import( - globals(), - aliases={ - "serve": ".legacy.server", - "unix_serve": ".legacy.server", - "WebSocketServerProtocol": ".legacy.server", - "WebSocketServer": ".legacy.server", - }, -) +# See #940 for why lazy_import isn't used here for backwards compatibility. +from .legacy.server import * # isort:skip # noqa __all__ = ["ServerConnection"] From 0713dbf2d37a8c2c071d8479a6768dd3d3c7dacf Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 14 May 2021 07:54:58 +0200 Subject: [PATCH 100/104] Add test coverage. --- tests/test_auth.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/test_auth.py diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 000000000..d5a8bd9ad --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1 @@ +from websockets.auth import * # noqa From 8900c13d3234c8ae87b0d852e849eaf6bf7cf8b7 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 15 May 2021 14:19:47 +0200 Subject: [PATCH 101/104] Add mypy to dictionary. --- docs/spelling_wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index d7c744147..4d8fc1e2d 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -31,6 +31,7 @@ lifecycle Lifecycle lookups MiB +mypy nginx parsers permessage From a14226afb77b524c2ced7d649ac7420a14992716 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 15 May 2021 18:01:23 +0200 Subject: [PATCH 102/104] Bump version number. --- docs/changelog.rst | 2 +- src/websockets/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 218bbec3d..1064af736 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -33,7 +33,7 @@ They may change at any time. 9.0.2 ..... -*In development* +*May 15, 2021* * Restored compatibility of ``python -m websockets`` with Python < 3.9. diff --git a/src/websockets/version.py b/src/websockets/version.py index 23b7f329b..02dbe9d3c 100644 --- a/src/websockets/version.py +++ b/src/websockets/version.py @@ -1 +1 @@ -version = "9.0.1" +version = "9.0.2" From 547a26b685d08cac0aa64e5e65f7867ac0ea9bc0 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 23 May 2021 18:51:27 +0200 Subject: [PATCH 103/104] Use constant-time comparison for passwords. Backport of c91b4c2a and dfecbd03. --- docs/changelog.rst | 6 ++++++ src/websockets/legacy/auth.py | 28 +++++++++++++++------------- tests/legacy/test_auth.py | 11 +++++++++-- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1064af736..f3e1acf08 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,6 +30,12 @@ They may change at any time. *In development* +.. note:: + + **Version 9.1 fixes a security issue introduced in version 8.0.** + + Version 8.0 was vulnerable to timing attacks on HTTP Basic Auth passwords. + 9.0.2 ..... diff --git a/src/websockets/legacy/auth.py b/src/websockets/legacy/auth.py index e0beede57..80ceff28d 100644 --- a/src/websockets/legacy/auth.py +++ b/src/websockets/legacy/auth.py @@ -6,6 +6,7 @@ import functools +import hmac import http from typing import Any, Awaitable, Callable, Iterable, Optional, Tuple, Union, cast @@ -132,24 +133,23 @@ def basic_auth_protocol_factory( if credentials is not None: if is_credentials(credentials): - - async def check_credentials(username: str, password: str) -> bool: - return (username, password) == credentials - + credentials_list = [cast(Credentials, credentials)] elif isinstance(credentials, Iterable): credentials_list = list(credentials) - if all(is_credentials(item) for item in credentials_list): - credentials_dict = dict(credentials_list) - - async def check_credentials(username: str, password: str) -> bool: - return credentials_dict.get(username) == password - - else: + if not all(is_credentials(item) for item in credentials_list): raise TypeError(f"invalid credentials argument: {credentials}") - else: raise TypeError(f"invalid credentials argument: {credentials}") + credentials_dict = dict(credentials_list) + + async def check_credentials(username: str, password: str) -> bool: + try: + expected_password = credentials_dict[username] + except KeyError: + return False + return hmac.compare_digest(expected_password, password) + if create_protocol is None: # Not sure why mypy cannot figure this out. create_protocol = cast( @@ -158,5 +158,7 @@ async def check_credentials(username: str, password: str) -> bool: ) return functools.partial( - create_protocol, realm=realm, check_credentials=check_credentials + create_protocol, + realm=realm, + check_credentials=check_credentials, ) diff --git a/tests/legacy/test_auth.py b/tests/legacy/test_auth.py index bb8c6a6eb..3d8eb90d7 100644 --- a/tests/legacy/test_auth.py +++ b/tests/legacy/test_auth.py @@ -1,3 +1,4 @@ +import hmac import unittest import urllib.error @@ -76,7 +77,7 @@ def test_basic_auth_bad_multiple_credentials(self): ) async def check_credentials(username, password): - return password == "iloveyou" + return hmac.compare_digest(password, "iloveyou") create_protocol_check_credentials = basic_auth_protocol_factory( realm="auth-tests", @@ -140,7 +141,13 @@ def test_basic_auth_unsupported_credentials_details(self): self.assertEqual(raised.exception.read().decode(), "Unsupported credentials\n") @with_server(create_protocol=create_protocol) - def test_basic_auth_invalid_credentials(self): + def test_basic_auth_invalid_username(self): + with self.assertRaises(InvalidStatusCode) as raised: + self.start_client(user_info=("goodbye", "iloveyou")) + self.assertEqual(raised.exception.status_code, 401) + + @with_server(create_protocol=create_protocol) + def test_basic_auth_invalid_password(self): with self.assertRaises(InvalidStatusCode) as raised: self.start_client(user_info=("hello", "ihateyou")) self.assertEqual(raised.exception.status_code, 401) From d0f328888f3e695aa64d78dcf48af4ece219221b Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Thu, 27 May 2021 13:32:46 +0200 Subject: [PATCH 104/104] Bump version number. --- docs/changelog.rst | 2 +- docs/conf.py | 4 ++-- src/websockets/version.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f3e1acf08..a82008a49 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,7 +28,7 @@ They may change at any time. 9.1 ... -*In development* +*May 27, 2021* .. note:: diff --git a/docs/conf.py b/docs/conf.py index dad7475f7..2246c0287 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = '9.0' +version = '9.1' # The full version, including alpha/beta/rc tags. -release = '9.0' +release = '9.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/websockets/version.py b/src/websockets/version.py index 02dbe9d3c..a7901ef92 100644 --- a/src/websockets/version.py +++ b/src/websockets/version.py @@ -1 +1 @@ -version = "9.0.2" +version = "9.1"