diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bcd42875..b747297c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,8 +43,8 @@ jobs: NVIM_BIN_PATH: nvim-linux64/bin EXTRACT: tar xzf - os: 'macos-latest' - NIGHTLY: nvim-macos.tar.gz - NVIM_BIN_PATH: nvim-macos/bin + NIGHTLY: nvim-macos-x86_64.tar.gz + NVIM_BIN_PATH: nvim-macos-x86_64/bin EXTRACT: tar xzf - os: 'windows-latest' NIGHTLY: nvim-win64.zip diff --git a/.readthedocs.yml b/.readthedocs.yml index d234062e..db303bdf 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -17,6 +17,8 @@ sphinx: fail_on_warning: true python: - install: - - method: pip - path: . + install: + - method: pip + path: . + extra_requirements: + - docs # pip install .[docs] diff --git a/README.md b/README.md index 26c4d908..a573a858 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ Pynvim defines some extensions over the vim python API: See the [Python Plugin API](http://pynvim.readthedocs.io/en/latest/usage/python-plugin-api.html) documentation for usage of this new functionality. +### Known Issues +- Vim evaluates `'v:'` to ``, whereas neovim evaluates to ``. This is expected behaviour due to the way booleans are implemented in python as explained [here](https://github.com/neovim/pynvim/issues/523#issuecomment-1495502011). + Development ----------- @@ -69,21 +72,22 @@ documentation. ### Usage from the Python REPL A number of different transports are supported, but the simplest way to get -started is with the python REPL. First, start Nvim with a known address (or use -the `$NVIM_LISTEN_ADDRESS` of a running instance): +started is with the python REPL. First, start Nvim with a known address: ```sh -$ NVIM_LISTEN_ADDRESS=/tmp/nvim nvim +$ nvim --listen /tmp/nvim.sock ``` +Or alternatively, note the `v:servername` address of a running Nvim instance. + In another terminal, connect a python REPL to Nvim (note that the API is similar to the one exposed by the [python-vim bridge](http://vimdoc.sourceforge.net/htmldoc/if_pyth.html#python-vim)): ```python >>> import pynvim -# Create a python API session attached to unix domain socket created above: ->>> nvim = pynvim.attach('socket', path='/tmp/nvim') +# Create a session attached to Nvim's address (`v:servername`). +>>> nvim = pynvim.attach('socket', path='/tmp/nvim.sock') # Now do some work. >>> buffer = nvim.current.buffer # Get the current buffer >>> buffer[0] = 'replace first line' diff --git a/docs/development.rst b/docs/development.rst index 43138d07..de08e769 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -7,7 +7,7 @@ If you change the code, you need to run:: for the changes to have effect. -Alternatively you could execute Neovim with the ``$PYTHONPATH`` environment variable:: +Alternatively you could execute Nvim with the ``$PYTHONPATH`` environment variable:: PYTHONPATH=/path/to/pynvim nvim @@ -15,13 +15,13 @@ But note this is not completely reliable, as installed packages can appear before ``$PYTHONPATH`` in the python search path. You need to rerun this command if you have changed the code, -in order for Neovim to use it for the plugin host. +in order for Nvim to use it for the plugin host. To run the tests execute:: python -m pytest -This will run the tests in an embedded instance of Neovim, with the current +This will run the tests in an embedded instance of Nvim, with the current directory added to ``sys.path``. If you want to test a different version than ``nvim`` in ``$PATH`` use:: @@ -30,11 +30,11 @@ If you want to test a different version than ``nvim`` in ``$PATH`` use:: Alternatively, if you want to see the state of nvim, you could use:: - export NVIM_LISTEN_ADDRESS=/tmp/nvimtest - xterm -e "nvim -u NONE"& + export NVIM=/tmp/nvimtest + xterm -e "nvim --listen $NVIM -u NONE" & python -m pytest -But note you need to restart Neovim every time you run the tests! +But note you need to restart Nvim every time you run the tests! Substitute your favorite terminal emulator for ``xterm``. Contributing @@ -58,7 +58,7 @@ If you have `tox`_, you can test with multiple python versions locally: Troubleshooting --------------- -You can run the plugin host in Neovim with logging enabled to debug errors:: +You can run the plugin host in Nvim with logging enabled to debug errors:: NVIM_PYTHON_LOG_FILE=logfile NVIM_PYTHON_LOG_LEVEL=DEBUG nvim @@ -75,18 +75,18 @@ Usage through the Python REPL A number of different transports are supported, but the simplest way to get started is with the python REPL. -First, start Neovim with a known address (or use the ``$NVIM_LISTEN_ADDRESS`` of a running instance):: +First, start Nvim with a known address (or use the ``v:servername`` of a running instance):: - NVIM_LISTEN_ADDRESS=/tmp/nvim nvim + nvim --listen /tmp/nvim.sock In another terminal, -connect a python REPL to Neovim (note that the API is similar to the one exposed by the `python-vim bridge`_): +connect a python REPL to Nvim (note that the API is similar to the one exposed by the `python-vim bridge`_): .. code-block:: python >>> from pynvim import attach - # Create a python API session attached to unix domain socket created above: - >>> nvim = attach('socket', path='/tmp/nvim') + # Create a session attached to Nvim's address (`v:servername`). + >>> nvim = attach('socket', path='/tmp/nvim.sock') # Now do some work. >>> buffer = nvim.current.buffer # Get the current buffer >>> buffer[0] = 'replace first line' @@ -99,7 +99,7 @@ connect a python REPL to Neovim (note that the API is similar to the one exposed .. _`python-vim bridge`: http://vimdoc.sourceforge.net/htmldoc/if_pyth.html#python-vim -You can embed Neovim into your python application instead of binding to a running neovim instance: +You can embed Nvim into your python application instead of binding to a running neovim instance: .. code-block:: python diff --git a/pynvim/_version.py b/pynvim/_version.py index e2927745..5d3c4712 100644 --- a/pynvim/_version.py +++ b/pynvim/_version.py @@ -4,7 +4,7 @@ from types import SimpleNamespace # see also setup.py -VERSION = SimpleNamespace(major=0, minor=5, patch=0, prerelease="") +VERSION = SimpleNamespace(major=0, minor=5, patch=1, prerelease="") # e.g. "0.5.0", "0.5.0.dev0" (PEP-440) __version__ = '{major}.{minor}.{patch}'.format(**vars(VERSION)) diff --git a/pynvim/api/buffer.py b/pynvim/api/buffer.py index ffadcb60..b9fb3bc6 100644 --- a/pynvim/api/buffer.py +++ b/pynvim/api/buffer.py @@ -1,4 +1,7 @@ """API for working with a Nvim Buffer.""" + +from __future__ import annotations + from typing import (Any, Iterator, List, Optional, TYPE_CHECKING, Tuple, Union, cast, overload) @@ -44,7 +47,7 @@ class Buffer(Remote): _api_prefix = "nvim_buf_" _session: "Nvim" - def __init__(self, session: "Nvim", code_data: Tuple[int, Any]): + def __init__(self, session: Nvim, code_data: Tuple[int, Any]): """Initialize from Nvim and code_data immutable object.""" super().__init__(session, code_data) @@ -150,7 +153,7 @@ def mark(self, name: str) -> Tuple[int, int]: """Return (row, col) tuple for a named mark.""" return cast(Tuple[int, int], tuple(self.request('nvim_buf_get_mark', name))) - def range(self, start: int, end: int) -> "Range": + def range(self, start: int, end: int) -> Range: """Return a `Range` object, which represents part of the Buffer.""" return Range(self, start, end) @@ -245,7 +248,8 @@ def number(self) -> int: return self.handle -class Range(object): +class Range: + def __init__(self, buffer: Buffer, start: int, end: int): self._buffer = buffer self.start = start - 1 diff --git a/pynvim/api/common.py b/pynvim/api/common.py index 3caab000..0e109a10 100644 --- a/pynvim/api/common.py +++ b/pynvim/api/common.py @@ -80,8 +80,7 @@ def request(self, name: str, *args: Any, **kwargs: Any) -> Any: return self._session.request(name, self, *args, **kwargs) -class RemoteApi(object): - +class RemoteApi: """Wrapper to allow api methods to be called like python methods.""" def __init__(self, obj: IRemote, api_prefix: str): @@ -106,7 +105,7 @@ def transform_keyerror(exc: E) -> Union[E, KeyError]: return exc -class RemoteMap(object): +class RemoteMap: """Represents a string->object map stored in Nvim. This is the dict counterpart to the `RemoteSequence` class, but it is used diff --git a/pynvim/api/nvim.py b/pynvim/api/nvim.py index b1b75eb8..f0c33fdc 100644 --- a/pynvim/api/nvim.py +++ b/pynvim/api/nvim.py @@ -1,4 +1,8 @@ """Main Nvim interface.""" + +from __future__ import annotations + +import asyncio import os import sys import threading @@ -54,8 +58,7 @@ """ -class Nvim(object): - +class Nvim: """Class that represents a remote Nvim instance. This class is main entry point to Nvim remote API, it is a wrapper @@ -79,7 +82,7 @@ class Nvim(object): """ @classmethod - def from_session(cls, session: 'Session') -> 'Nvim': + def from_session(cls, session: Session) -> Nvim: """Create a new Nvim instance for a Session instance. This method must be called to create the first Nvim instance, since it @@ -100,14 +103,14 @@ def from_session(cls, session: 'Session') -> 'Nvim': return cls(session, channel_id, metadata, types) @classmethod - def from_nvim(cls, nvim: 'Nvim') -> 'Nvim': + def from_nvim(cls, nvim: Nvim) -> Nvim: """Create a new Nvim instance from an existing instance.""" return cls(nvim._session, nvim.channel_id, nvim.metadata, nvim.types, nvim._decode, nvim._err_cb) def __init__( self, - session: 'Session', + session: Session, channel_id: int, metadata: Dict[str, Any], types: Dict[int, Any], @@ -140,7 +143,16 @@ def __init__( self._err_cb: Callable[[str], Any] = lambda _: None else: self._err_cb = err_cb - self.loop = self._session.loop._loop + + @property + def loop(self) -> asyncio.AbstractEventLoop: + """Get the event loop (exposed to rplugins).""" # noqa + + # see #294: for python 3.4+, the only available and guaranteed + # implementation of msgpack_rpc BaseEventLoop is the AsyncioEventLoop. + # The underlying asyncio event loop is exposed to rplugins. + # pylint: disable=protected-access + return self._session.loop._loop # type: ignore def _from_nvim(self, obj: Any, decode: Optional[TDecodeMode] = None) -> Any: if decode is None: @@ -157,7 +169,7 @@ def _to_nvim(self, obj: Any) -> Any: return ExtType(*obj.code_data) return obj - def _get_lua_private(self) -> 'LuaFuncs': + def _get_lua_private(self) -> LuaFuncs: if not getattr(self._session, "_has_lua", False): self.exec_lua(lua_module, self.channel_id) self._session._has_lua = True # type: ignore[attr-defined] @@ -258,7 +270,7 @@ def close(self) -> None: """Close the nvim session and release its resources.""" self._session.close() - def __enter__(self) -> 'Nvim': + def __enter__(self) -> Nvim: """Enter nvim session as a context manager.""" return self @@ -269,7 +281,7 @@ def __exit__(self, *exc_info: Any) -> None: """ self.close() - def with_decode(self, decode: Literal[True] = True) -> 'Nvim': + def with_decode(self, decode: Literal[True] = True) -> Nvim: """Initialize a new Nvim instance.""" return Nvim(self._session, self.channel_id, self.metadata, self.types, decode, self._err_cb) @@ -564,8 +576,7 @@ def tabpage(self, tabpage: Union[Tabpage, int]) -> None: return self._session.request('nvim_set_current_tabpage', tabpage) -class Funcs(object): - +class Funcs: """Helper class for functional vimscript interface.""" def __init__(self, nvim: Nvim): @@ -575,15 +586,14 @@ def __getattr__(self, name: str) -> Callable[..., Any]: return partial(self._nvim.call, name) -class LuaFuncs(object): - +class LuaFuncs: """Wrapper to allow lua functions to be called like python methods.""" def __init__(self, nvim: Nvim, name: str = ""): self._nvim = nvim self.name = name - def __getattr__(self, name: str) -> 'LuaFuncs': + def __getattr__(self, name: str) -> LuaFuncs: """Return wrapper to named api method.""" prefix = self.name + "." if self.name else "" return LuaFuncs(self._nvim, prefix + name) diff --git a/pynvim/api/tabpage.py b/pynvim/api/tabpage.py index 244be9d9..f6ee28ec 100644 --- a/pynvim/api/tabpage.py +++ b/pynvim/api/tabpage.py @@ -1,8 +1,12 @@ """API for working with Nvim tabpages.""" + +from __future__ import annotations + from typing import Any, TYPE_CHECKING, Tuple from pynvim.api.common import Remote, RemoteSequence from pynvim.api.window import Window + if TYPE_CHECKING: from pynvim.api.nvim import Nvim @@ -15,7 +19,7 @@ class Tabpage(Remote): _api_prefix = "nvim_tabpage_" - def __init__(self, session: 'Nvim', code_data: Tuple[int, Any]): + def __init__(self, session: Nvim, code_data: Tuple[int, Any]): """Initialize from session and code_data immutable object. The `code_data` contains serialization information required for diff --git a/pynvim/api/window.py b/pynvim/api/window.py index d0b82903..8ad26e48 100644 --- a/pynvim/api/window.py +++ b/pynvim/api/window.py @@ -1,4 +1,7 @@ """API for working with Nvim windows.""" + +from __future__ import annotations + from typing import TYPE_CHECKING, Tuple, cast from pynvim.api.buffer import Buffer @@ -63,7 +66,7 @@ def col(self) -> int: return self.request('nvim_win_get_position')[1] @property - def tabpage(self) -> 'Tabpage': + def tabpage(self) -> Tabpage: """Get the `Tabpage` that contains the window.""" return self.request('nvim_win_get_tabpage') diff --git a/pynvim/msgpack_rpc/async_session.py b/pynvim/msgpack_rpc/async_session.py index 333c6cf2..e7766454 100644 --- a/pynvim/msgpack_rpc/async_session.py +++ b/pynvim/msgpack_rpc/async_session.py @@ -1,13 +1,19 @@ """Asynchronous msgpack-rpc handling in the event loop pipeline.""" import logging from traceback import format_exc +from typing import Any, AnyStr, Callable, Dict +from pynvim.msgpack_rpc.msgpack_stream import MsgpackStream logger = logging.getLogger(__name__) debug, info, warn = (logger.debug, logger.info, logger.warning,) -class AsyncSession(object): +# response call back takes two arguments: (err, return_value) +ResponseCallback = Callable[..., None] + + +class AsyncSession: """Asynchronous msgpack-rpc layer that wraps a msgpack stream. @@ -16,11 +22,11 @@ class AsyncSession(object): requests and notifications. """ - def __init__(self, msgpack_stream): + def __init__(self, msgpack_stream: MsgpackStream): """Wrap `msgpack_stream` on a msgpack-rpc interface.""" self._msgpack_stream = msgpack_stream self._next_request_id = 1 - self._pending_requests = {} + self._pending_requests: Dict[int, ResponseCallback] = {} self._request_cb = self._notification_cb = None self._handlers = { 0: self._on_request, @@ -33,7 +39,8 @@ def threadsafe_call(self, fn): """Wrapper around `MsgpackStream.threadsafe_call`.""" self._msgpack_stream.threadsafe_call(fn) - def request(self, method, args, response_cb): + def request(self, method: AnyStr, args: Any, + response_cb: ResponseCallback) -> None: """Send a msgpack-rpc request to Nvim. A msgpack-rpc with method `method` and argument `args` is sent to @@ -89,8 +96,9 @@ def _on_request(self, msg): # - msg[2]: method name # - msg[3]: arguments debug('received request: %s, %s', msg[2], msg[3]) - self._request_cb(msg[2], msg[3], Response(self._msgpack_stream, - msg[1])) + assert self._request_cb is not None + self._request_cb(msg[2], msg[3], + Response(self._msgpack_stream, msg[1])) def _on_response(self, msg): # response to a previous request: @@ -105,6 +113,7 @@ def _on_notification(self, msg): # - msg[1]: event name # - msg[2]: arguments debug('received notification: %s, %s', msg[1], msg[2]) + assert self._notification_cb is not None self._notification_cb(msg[1], msg[2]) def _on_invalid_message(self, msg): @@ -113,15 +122,14 @@ def _on_invalid_message(self, msg): self._msgpack_stream.send([1, 0, error, None]) -class Response(object): - +class Response: """Response to a msgpack-rpc request that came from Nvim. When Nvim sends a msgpack-rpc request, an instance of this class is created for remembering state required to send a response. """ - def __init__(self, msgpack_stream, request_id): + def __init__(self, msgpack_stream: MsgpackStream, request_id: int): """Initialize the Response instance.""" self._msgpack_stream = msgpack_stream self._request_id = request_id diff --git a/pynvim/msgpack_rpc/event_loop/__init__.py b/pynvim/msgpack_rpc/event_loop/__init__.py index e94cdbfe..1cf40a77 100644 --- a/pynvim/msgpack_rpc/event_loop/__init__.py +++ b/pynvim/msgpack_rpc/event_loop/__init__.py @@ -1,6 +1,6 @@ """Event loop abstraction subpackage. -Tries to use pyuv as a backend, falling back to the asyncio implementation. +We use python's built-in asyncio as the backend. """ from pynvim.msgpack_rpc.event_loop.asyncio import AsyncioEventLoop as EventLoop diff --git a/pynvim/msgpack_rpc/event_loop/asyncio.py b/pynvim/msgpack_rpc/event_loop/asyncio.py index 164173b8..cb17f321 100644 --- a/pynvim/msgpack_rpc/event_loop/asyncio.py +++ b/pynvim/msgpack_rpc/event_loop/asyncio.py @@ -1,12 +1,6 @@ -"""Event loop implementation that uses the `asyncio` standard module. +"""Event loop implementation that uses the `asyncio` standard module.""" -The `asyncio` module was added to python standard library on 3.4, and it -provides a pure python implementation of an event loop library. It is used -as a fallback in case pyuv is not available(on python implementations other -than CPython). - -""" -from __future__ import absolute_import +from __future__ import annotations import asyncio import logging @@ -14,17 +8,23 @@ import sys from collections import deque from signal import Signals -from typing import Any, Callable, Deque, List, Optional +from typing import Any, Callable, Deque, List, Optional, cast + +if sys.version_info >= (3, 12): + from typing import Final, override +else: + from typing_extensions import Final, override -from pynvim.msgpack_rpc.event_loop.base import BaseEventLoop +from pynvim.msgpack_rpc.event_loop.base import BaseEventLoop, TTransportType logger = logging.getLogger(__name__) debug, info, warn = (logger.debug, logger.info, logger.warning,) loop_cls = asyncio.SelectorEventLoop + if os.name == 'nt': + import msvcrt # pylint: disable=import-error from asyncio.windows_utils import PipeHandle # type: ignore[attr-defined] - import msvcrt # On windows use ProactorEventLoop which support pipes and is backed by the # more powerful IOCP facility @@ -32,134 +32,249 @@ loop_cls = asyncio.ProactorEventLoop # type: ignore[attr-defined,misc] -class AsyncioEventLoop(BaseEventLoop, asyncio.Protocol, - asyncio.SubprocessProtocol): - """`BaseEventLoop` subclass that uses `asyncio` as a backend.""" +# pylint: disable=logging-fstring-interpolation - _queued_data: Deque[bytes] - if os.name != 'nt': - _child_watcher: Optional['asyncio.AbstractChildWatcher'] +class Protocol(asyncio.Protocol, asyncio.SubprocessProtocol): + """The protocol class used for asyncio-based RPC communication.""" + + def __init__(self, on_data, on_error): + """Initialize the Protocol object.""" + assert on_data is not None + assert on_error is not None + self._on_data = on_data + self._on_error = on_error + @override def connection_made(self, transport): """Used to signal `asyncio.Protocol` of a successful connection.""" - self._transport = transport - self._raw_transport = transport - if isinstance(transport, asyncio.SubprocessTransport): - self._transport = transport.get_pipe_transport(0) + del transport # no-op - def connection_lost(self, exc): + @override + def connection_lost(self, exc: Optional[Exception]) -> None: """Used to signal `asyncio.Protocol` of a lost connection.""" - self._on_error(exc.args[0] if exc else 'EOF') + debug(f"connection_lost: exc = {exc}") + self._on_error(exc if exc else EOFError()) + @override def data_received(self, data: bytes) -> None: """Used to signal `asyncio.Protocol` of incoming data.""" - if self._on_data: - self._on_data(data) - return - self._queued_data.append(data) + self._on_data(data) - def pipe_connection_lost(self, fd, exc): + @override + def pipe_connection_lost(self, fd: int, exc: Optional[Exception]) -> None: """Used to signal `asyncio.SubprocessProtocol` of a lost connection.""" debug("pipe_connection_lost: fd = %s, exc = %s", fd, exc) if os.name == 'nt' and fd == 2: # stderr # On windows, ignore piped stderr being closed immediately (#505) return - self._on_error(exc.args[0] if exc else 'EOF') + self._on_error(exc if exc else EOFError()) + @override def pipe_data_received(self, fd, data): """Used to signal `asyncio.SubprocessProtocol` of incoming data.""" if fd == 2: # stderr fd number # Ignore stderr message, log only for debugging debug("stderr: %s", str(data)) - elif self._on_data: - self._on_data(data) - else: - self._queued_data.append(data) + elif fd == 1: # stdout + self.data_received(data) + @override def process_exited(self) -> None: """Used to signal `asyncio.SubprocessProtocol` when the child exits.""" - self._on_error('EOF') + debug("process_exited") + self._on_error(EOFError()) + + +class AsyncioEventLoop(BaseEventLoop): + """`BaseEventLoop` subclass that uses core `asyncio` as a backend.""" + + _protocol: Optional[Protocol] + _transport: Optional[asyncio.WriteTransport] + _signals: List[Signals] + _data_buffer: Deque[bytes] + if os.name != 'nt': + _child_watcher: Optional[asyncio.AbstractChildWatcher] + + def __init__(self, + transport_type: TTransportType, + *args: Any, **kwargs: Any): + """asyncio-specific initialization. see BaseEventLoop.__init__.""" + + # The underlying asyncio event loop. + self._loop: Final[asyncio.AbstractEventLoop] = loop_cls() + + # Handle messages from nvim that may arrive before run() starts. + self._data_buffer = deque() + + def _on_data(data: bytes) -> None: + if self._on_data is None: + self._data_buffer.append(data) + return + self._on_data(data) - def _init(self) -> None: - self._loop = loop_cls() - self._queued_data = deque() - self._fact = lambda: self - self._raw_transport = None + # pylint: disable-next=unnecessary-lambda + self._protocol_factory = lambda: Protocol( + on_data=_on_data, + on_error=self._on_error, + ) + self._protocol = None + + # The communication channel (endpoint) created by _connect_*() methods, + # where we write request messages to be sent to neovim + self._transport = None + self._to_close: List[asyncio.BaseTransport] = [] self._child_watcher = None + super().__init__(transport_type, *args, **kwargs) + + @override def _connect_tcp(self, address: str, port: int) -> None: - coroutine = self._loop.create_connection(self._fact, address, port) - self._loop.run_until_complete(coroutine) + async def connect_tcp(): + transport, protocol = await self._loop.create_connection( + self._protocol_factory, address, port) + debug(f"tcp connection successful: {address}:{port}") + self._transport = transport + self._protocol = protocol + + self._loop.run_until_complete(connect_tcp()) + @override def _connect_socket(self, path: str) -> None: - if os.name == 'nt': - coroutine = self._loop.create_pipe_connection( # type: ignore[attr-defined] - self._fact, path - ) - else: - coroutine = self._loop.create_unix_connection(self._fact, path) - self._loop.run_until_complete(coroutine) + async def connect_socket(): + if os.name == 'nt': + _create_connection = self._loop.create_pipe_connection + else: + _create_connection = self._loop.create_unix_connection + + transport, protocol = await _create_connection( + self._protocol_factory, path) + debug("socket connection successful: %s", self._transport) + self._transport = transport + self._protocol = protocol + self._loop.run_until_complete(connect_socket()) + + @override def _connect_stdio(self) -> None: - if os.name == 'nt': - pipe: Any = PipeHandle( - msvcrt.get_osfhandle(sys.stdin.fileno()) # type: ignore[attr-defined] - ) - else: - pipe = sys.stdin - coroutine = self._loop.connect_read_pipe(self._fact, pipe) - self._loop.run_until_complete(coroutine) - debug("native stdin connection successful") + async def connect_stdin(): + if os.name == 'nt': + pipe = PipeHandle(msvcrt.get_osfhandle(sys.stdin.fileno())) + else: + pipe = sys.stdin + transport, protocol = await self._loop.connect_read_pipe( + self._protocol_factory, pipe) + debug("native stdin connection successful") + self._to_close.append(transport) + del protocol + self._loop.run_until_complete(connect_stdin()) # Make sure subprocesses don't clobber stdout, # send the output to stderr instead. rename_stdout = os.dup(sys.stdout.fileno()) os.dup2(sys.stderr.fileno(), sys.stdout.fileno()) - if os.name == 'nt': - pipe = PipeHandle( - msvcrt.get_osfhandle(rename_stdout) # type: ignore[attr-defined] - ) - else: - pipe = os.fdopen(rename_stdout, 'wb') - coroutine = self._loop.connect_write_pipe(self._fact, pipe) # type: ignore[assignment] - self._loop.run_until_complete(coroutine) - debug("native stdout connection successful") + async def connect_stdout(): + if os.name == 'nt': + pipe = PipeHandle(msvcrt.get_osfhandle(rename_stdout)) + else: + pipe = os.fdopen(rename_stdout, 'wb') + + transport, protocol = await self._loop.connect_write_pipe( + self._protocol_factory, pipe) + debug("native stdout connection successful") + self._transport = transport + self._protocol = protocol + self._loop.run_until_complete(connect_stdout()) + @override def _connect_child(self, argv: List[str]) -> None: if os.name != 'nt': + # see #238, #241 self._child_watcher = asyncio.get_child_watcher() self._child_watcher.attach_loop(self._loop) - coroutine = self._loop.subprocess_exec(self._fact, *argv) - self._loop.run_until_complete(coroutine) + async def create_subprocess(): + transport: asyncio.SubprocessTransport # type: ignore + transport, protocol = await self._loop.subprocess_exec( + self._protocol_factory, *argv) + pid = transport.get_pid() + debug("child subprocess_exec successful, PID = %s", pid) + + self._transport = cast(asyncio.WriteTransport, + transport.get_pipe_transport(0)) # stdin + self._protocol = protocol + + # proactor transport implementations do not close the pipes + # automatically, so make sure they are closed upon shutdown + def _close_later(transport): + if transport is not None: + self._to_close.append(transport) + + _close_later(transport.get_pipe_transport(1)) + _close_later(transport.get_pipe_transport(2)) + _close_later(transport) + + # await until child process have been launched and the transport has + # been established + self._loop.run_until_complete(create_subprocess()) + + @override def _start_reading(self) -> None: pass + @override def _send(self, data: bytes) -> None: + assert self._transport, "connection has not been established." self._transport.write(data) + @override def _run(self) -> None: - while self._queued_data: - data = self._queued_data.popleft() + # process the early messages that arrived as soon as the transport + # channels are open and on_data is fully ready to receive messages. + while self._data_buffer: + data: bytes = self._data_buffer.popleft() if self._on_data is not None: self._on_data(data) + self._loop.run_forever() + @override def _stop(self) -> None: self._loop.stop() + @override def _close(self) -> None: - if self._raw_transport is not None: - self._raw_transport.close() + def _close_transport(transport): + transport.close() + + # Windows: for ProactorBasePipeTransport, close() doesn't take in + # effect immediately (closing happens asynchronously inside the + # event loop), need to wait a bit for completing graceful shutdown. + if os.name == 'nt' and hasattr(transport, '_sock'): + async def wait_until_closed(): + # pylint: disable-next=protected-access + while transport._sock is not None: + await asyncio.sleep(0.01) + self._loop.run_until_complete(wait_until_closed()) + + if self._transport: + _close_transport(self._transport) + self._transport = None + for transport in self._to_close: + _close_transport(transport) + self._to_close[:] = [] + self._loop.close() + if self._child_watcher is not None: self._child_watcher.close() self._child_watcher = None + @override def _threadsafe_call(self, fn: Callable[[], Any]) -> None: self._loop.call_soon_threadsafe(fn) + @override def _setup_signals(self, signals: List[Signals]) -> None: if os.name == 'nt': # add_signal_handler is not supported in win32 @@ -170,6 +285,7 @@ def _setup_signals(self, signals: List[Signals]) -> None: for signum in self._signals: self._loop.add_signal_handler(signum, self._on_signal, signum) + @override def _teardown_signals(self) -> None: for signum in self._signals: self._loop.remove_signal_handler(signum) diff --git a/pynvim/msgpack_rpc/event_loop/base.py b/pynvim/msgpack_rpc/event_loop/base.py index 86fde9c2..c7def3e1 100644 --- a/pynvim/msgpack_rpc/event_loop/base.py +++ b/pynvim/msgpack_rpc/event_loop/base.py @@ -4,7 +4,7 @@ import sys import threading from abc import ABC, abstractmethod -from typing import Any, Callable, List, Optional, Type, Union +from typing import Any, Callable, List, Optional, Union if sys.version_info < (3, 8): from typing_extensions import Literal @@ -28,15 +28,28 @@ Literal['child'] ] +# TODO: Since pynvim now supports python 3, the only available backend of the +# msgpack_rpc BaseEventLoop is the built-in asyncio (see #294). We will have +# to remove some unnecessary abstractions as well as greenlet. See also #489 -class BaseEventLoop(ABC): +class BaseEventLoop(ABC): """Abstract base class for all event loops. Event loops act as the bottom layer for Nvim sessions created by this library. They hide system/transport details behind a simple interface for reading/writing bytes to the connected Nvim instance. + A lifecycle of event loop is as follows: (1. -> [2. -> 3.]* -> 4.) + 1. initialization (__init__): connection to Nvim is established. + 2. run(data_cb): run the event loop (blocks until the loop stops). + Requests are sent to the remote neovim by calling send(), and + responses (messages) from the remote neovim will be passed to the + given `data_cb` callback function while the event loop is running. + Note that run() may be called multiple times. + 3. stop(): stop the event loop. + 4. close(): close the event loop, destroying all the internal resources. + This class exposes public methods for interacting with the underlying event loop and delegates implementation-specific work to the following methods, which subclasses are expected to implement: @@ -50,15 +63,17 @@ class BaseEventLoop(ABC): embedded Nvim that has its stdin/stdout connected to the event loop. - `_start_reading()`: Called after any of _connect_* methods. Can be used to perform any post-connection setup or validation. - - `_send(data)`: Send `data`(byte array) to Nvim. The data is only + - `_send(data)`: Send `data` (byte array) to Nvim (usually RPC request). - `_run()`: Runs the event loop until stopped or the connection is closed. - calling the following methods when some event happens: - actually sent when the event loop is running. - - `_on_data(data)`: When Nvim sends some data. + The following methods can be called upon some events by the event loop: + - `_on_data(data)`: When Nvim sends some data (usually RPC response). - `_on_signal(signum)`: When a signal is received. - - `_on_error(message)`: When a non-recoverable error occurs(eg: - connection lost) - - `_stop()`: Stop the event loop + - `_on_error(exc)`: When a non-recoverable error occurs (e.g: + connection lost, or any other OSError) + Note that these _on_{data,signal,error} methods are not 'final', may be + changed around an execution of run(). The subclasses are expected to + handle any early messages arriving while _on_data is not yet set. + - `_stop()`: Stop the event loop. - `_interrupt(data)`: Like `stop()`, but may be called from other threads this. - `_setup_signals(signals)`: Add implementation-specific listeners for @@ -73,33 +88,20 @@ def __init__(self, transport_type: TTransportType, *args: Any, **kwargs: Any): configuration, like this: >>> BaseEventLoop('tcp', '127.0.0.1', 7450) - Traceback (most recent call last): - ... - AttributeError: 'BaseEventLoop' object has no attribute '_init' >>> BaseEventLoop('socket', '/tmp/nvim-socket') - Traceback (most recent call last): - ... - AttributeError: 'BaseEventLoop' object has no attribute '_init' >>> BaseEventLoop('stdio') - Traceback (most recent call last): - ... - AttributeError: 'BaseEventLoop' object has no attribute '_init' - >>> BaseEventLoop('child', - ['nvim', '--embed', '--headless', '-u', 'NONE']) - Traceback (most recent call last): - ... - AttributeError: 'BaseEventLoop' object has no attribute '_init' - - This calls the implementation-specific initialization - `_init`, one of the `_connect_*` methods(based on `transport_type`) - and `_start_reading()` + >>> BaseEventLoop('child', ['nvim', '--embed', '--headless', '-u', 'NONE']) + + Implementation-specific initialization should be made in the __init__ + constructor of the subclass, which must call the constructor of the + super class (BaseEventLoop), in which one of the `_connect_*` methods + (based on `transport_type`) and then `_start_reading()`. """ self._transport_type = transport_type self._signames = dict((k, v) for v, k in signal.__dict__.items() if v.startswith('SIG')) self._on_data: Optional[Callable[[bytes], None]] = None self._error: Optional[BaseException] = None - self._init() try: getattr(self, '_connect_{}'.format(transport_type))(*args, **kwargs) except Exception as e: @@ -107,10 +109,6 @@ def __init__(self, transport_type: TTransportType, *args: Any, **kwargs: Any): raise e self._start_reading() - @abstractmethod - def _init(self) -> None: - raise NotImplementedError() - @abstractmethod def _start_reading(self) -> None: raise NotImplementedError() @@ -168,17 +166,23 @@ def threadsafe_call(self, fn): """ self._threadsafe_call(fn) - def run(self, data_cb): - """Run the event loop.""" + @abstractmethod + def _threadsafe_call(self, fn: Callable[[], Any]) -> None: + raise NotImplementedError() + + def run(self, data_cb: Callable[[bytes], None]) -> None: + """Run the event loop, and receives response messages to a callback.""" if self._error: err = self._error if isinstance(self._error, KeyboardInterrupt): - # KeyboardInterrupt is not destructive(it may be used in + # KeyboardInterrupt is not destructive (it may be used in # the REPL). # After throwing KeyboardInterrupt, cleanup the _error field # so the loop may be started again self._error = None raise err + + # data_cb: e.g., MsgpackStream._on_data self._on_data = data_cb if threading.current_thread() == main_thread: self._setup_signals([signal.SIGINT, signal.SIGTERM]) @@ -190,6 +194,10 @@ def run(self, data_cb): signal.signal(signal.SIGINT, default_int_handler) self._on_data = None + @abstractmethod + def _run(self) -> None: + raise NotImplementedError() + def stop(self) -> None: """Stop the event loop.""" self._stop() @@ -209,23 +217,32 @@ def _close(self) -> None: raise NotImplementedError() def _on_signal(self, signum: signal.Signals) -> None: - msg = 'Received {}'.format(self._signames[signum]) + # pylint: disable-next=consider-using-f-string + msg = 'Received signal {}'.format(self._signames[signum]) debug(msg) + if signum == signal.SIGINT and self._transport_type == 'stdio': # When the transport is stdio, we are probably running as a Nvim # child process. In that case, we don't want to be killed by # ctrl+C return - cls: Type[BaseException] = Exception + if signum == signal.SIGINT: - cls = KeyboardInterrupt - self._error = cls(msg) + self._error = KeyboardInterrupt() + else: + self._error = Exception(msg) self.stop() - def _on_error(self, error: str) -> None: - debug(error) - self._error = OSError(error) + def _on_error(self, exc: Exception) -> None: + debug(str(exc)) + self._error = exc self.stop() def _on_interrupt(self) -> None: self.stop() + + def _setup_signals(self, signals: List[signal.Signals]) -> None: + pass # no-op by default + + def _teardown_signals(self) -> None: + pass # no-op by default diff --git a/pynvim/msgpack_rpc/event_loop/uv.py b/pynvim/msgpack_rpc/event_loop/uv.py deleted file mode 100644 index 969187ee..00000000 --- a/pynvim/msgpack_rpc/event_loop/uv.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Event loop implementation that uses pyuv(libuv-python bindings).""" -import sys -from collections import deque - -import pyuv - -from pynvim.msgpack_rpc.event_loop.base import BaseEventLoop - - -class UvEventLoop(BaseEventLoop): - - """`BaseEventLoop` subclass that uses `pvuv` as a backend.""" - - def _init(self): - self._loop = pyuv.Loop() - self._async = pyuv.Async(self._loop, self._on_async) - self._connection_error = None - self._error_stream = None - self._callbacks = deque() - - def _on_connect(self, stream, error): - self.stop() - if error: - msg = 'Cannot connect to {}: {}'.format( - self._connect_address, pyuv.errno.strerror(error)) - self._connection_error = OSError(msg) - return - self._read_stream = self._write_stream = stream - - def _on_read(self, handle, data, error): - if error or not data: - msg = pyuv.errno.strerror(error) if error else 'EOF' - self._on_error(msg) - return - if handle == self._error_stream: - return - self._on_data(data) - - def _on_write(self, handle, error): - if error: - msg = pyuv.errno.strerror(error) - self._on_error(msg) - - def _on_exit(self, handle, exit_status, term_signal): - self._on_error('EOF') - - def _disconnected(self, *args): - raise OSError('Not connected to Nvim') - - def _connect_tcp(self, address, port): - stream = pyuv.TCP(self._loop) - self._connect_address = '{}:{}'.format(address, port) - stream.connect((address, port), self._on_connect) - - def _connect_socket(self, path): - stream = pyuv.Pipe(self._loop) - self._connect_address = path - stream.connect(path, self._on_connect) - - def _connect_stdio(self): - self._read_stream = pyuv.Pipe(self._loop) - self._read_stream.open(sys.stdin.fileno()) - self._write_stream = pyuv.Pipe(self._loop) - self._write_stream.open(sys.stdout.fileno()) - - def _connect_child(self, argv): - self._write_stream = pyuv.Pipe(self._loop) - self._read_stream = pyuv.Pipe(self._loop) - self._error_stream = pyuv.Pipe(self._loop) - stdin = pyuv.StdIO(self._write_stream, - flags=pyuv.UV_CREATE_PIPE + pyuv.UV_READABLE_PIPE) - stdout = pyuv.StdIO(self._read_stream, - flags=pyuv.UV_CREATE_PIPE + pyuv.UV_WRITABLE_PIPE) - stderr = pyuv.StdIO(self._error_stream, - flags=pyuv.UV_CREATE_PIPE + pyuv.UV_WRITABLE_PIPE) - pyuv.Process.spawn(self._loop, - args=argv, - exit_callback=self._on_exit, - flags=pyuv.UV_PROCESS_WINDOWS_HIDE, - stdio=(stdin, stdout, stderr,)) - self._error_stream.start_read(self._on_read) - - def _start_reading(self): - if self._transport_type in ['tcp', 'socket']: - self._loop.run() - if self._connection_error: - self.run = self.send = self._disconnected - raise self._connection_error - self._read_stream.start_read(self._on_read) - - def _send(self, data): - self._write_stream.write(data, self._on_write) - - def _run(self): - self._loop.run(pyuv.UV_RUN_DEFAULT) - - def _stop(self): - self._loop.stop() - - def _close(self): - pass - - def _threadsafe_call(self, fn): - self._callbacks.append(fn) - self._async.send() - - def _on_async(self, handle): - while self._callbacks: - self._callbacks.popleft()() - - def _setup_signals(self, signals): - self._signal_handles = [] - - def handler(h, signum): - self._on_signal(signum) - - for signum in signals: - handle = pyuv.Signal(self._loop) - handle.start(handler, signum) - self._signal_handles.append(handle) - - def _teardown_signals(self): - for handle in self._signal_handles: - handle.stop() diff --git a/pynvim/msgpack_rpc/msgpack_stream.py b/pynvim/msgpack_rpc/msgpack_stream.py index 49340c50..f209d849 100644 --- a/pynvim/msgpack_rpc/msgpack_stream.py +++ b/pynvim/msgpack_rpc/msgpack_stream.py @@ -4,20 +4,20 @@ from msgpack import Packer, Unpacker from pynvim.compat import unicode_errors_default +from pynvim.msgpack_rpc.event_loop.base import BaseEventLoop logger = logging.getLogger(__name__) debug, info, warn = (logger.debug, logger.info, logger.warning,) -class MsgpackStream(object): - +class MsgpackStream: """Two-way msgpack stream that wraps a event loop byte stream. This wraps the event loop interface for reading/writing bytes and exposes an interface for reading/writing msgpack documents. """ - def __init__(self, event_loop): + def __init__(self, event_loop: BaseEventLoop) -> None: """Wrap `event_loop` on a msgpack-aware interface.""" self.loop = event_loop self._packer = Packer(unicode_errors=unicode_errors_default) @@ -30,7 +30,7 @@ def threadsafe_call(self, fn): def send(self, msg): """Queue `msg` for sending to Nvim.""" - debug('sent %s', msg) + debug('sending %s', msg) self.loop.send(self._packer.pack(msg)) def run(self, message_cb): @@ -51,14 +51,15 @@ def close(self): """Close the event loop.""" self.loop.close() - def _on_data(self, data): + def _on_data(self, data: bytes) -> None: self._unpacker.feed(data) while True: try: debug('waiting for message...') msg = next(self._unpacker) debug('received message: %s', msg) - self._message_cb(msg) + assert self._message_cb is not None + self._message_cb(msg) # type: ignore[unreachable] except StopIteration: debug('unpacker needs more data...') break diff --git a/pynvim/msgpack_rpc/session.py b/pynvim/msgpack_rpc/session.py index e578f911..1c8e6f27 100644 --- a/pynvim/msgpack_rpc/session.py +++ b/pynvim/msgpack_rpc/session.py @@ -11,6 +11,7 @@ from pynvim.compat import check_async from pynvim.msgpack_rpc.async_session import AsyncSession +from pynvim.msgpack_rpc.event_loop.base import BaseEventLoop if sys.version_info < (3, 8): from typing_extensions import Literal @@ -42,7 +43,7 @@ class Notification(NamedTuple): Message = Union[Request, Notification] -class Session(object): +class Session: """Msgpack-rpc session layer that uses coroutines for a synchronous API. @@ -59,11 +60,15 @@ def __init__(self, async_session: AsyncSession): self._pending_messages: Deque[Message] = deque() self._is_running = False self._setup_exception: Optional[Exception] = None - self.loop = async_session.loop self._loop_thread: Optional[threading.Thread] = None self.error_wrapper: Callable[[Tuple[int, str]], Exception] = \ lambda e: Exception(e[1]) + @property + def loop(self) -> BaseEventLoop: + """Get the underlying msgpack EventLoop.""" + return self._async_session.loop + def threadsafe_call( self, fn: Callable[..., Any], *args: Any, **kwargs: Any ) -> None: diff --git a/pynvim/plugin/host.py b/pynvim/plugin/host.py index ea4c1df6..4a5a209b 100644 --- a/pynvim/plugin/host.py +++ b/pynvim/plugin/host.py @@ -6,6 +6,7 @@ import logging import os import os.path +import pathlib import re import sys from functools import partial @@ -173,7 +174,7 @@ def _load(self, plugins: Sequence[str]) -> None: # self.nvim.err_write("host init _load\n", async_=True) has_script = False for path in plugins: - path = os.path.normpath(path) # normalize path + path = pathlib.Path(os.path.normpath(path)).as_posix() # normalize path err = None if path in self._loaded: warn('{} is already loaded'.format(path)) @@ -276,6 +277,7 @@ def _copy_attributes(self, fn, fn2): def _on_specs_request(self, path): path = decode_if_bytes(path) + path = pathlib.Path(os.path.normpath(path)).as_posix() # normalize path if path in self._load_errors: self.nvim.out_write(self._load_errors[path] + '\n') return self._specs.get(path, 0) diff --git a/pynvim/plugin/script_host.py b/pynvim/plugin/script_host.py index f647e5e3..89efbadd 100644 --- a/pynvim/plugin/script_host.py +++ b/pynvim/plugin/script_host.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) -debug, info, warn = (logger.debug, logger.info, logger.warn,) +debug, info, warn = (logger.debug, logger.info, logger.warning,) @plugin @@ -195,7 +195,7 @@ def writelines(self, seq): def num_to_str(obj): - if isinstance(obj, num_types) and not isinstance(obj, bool): + if isinstance(obj, num_types): return str(obj) else: return obj diff --git a/scripts/logging_statement_modifier.py b/scripts/logging_statement_modifier.py index 57a194f0..0f3c4920 100755 --- a/scripts/logging_statement_modifier.py +++ b/scripts/logging_statement_modifier.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """\ Logging Statement Modifier - replace logging calls with pass (or vice versa) diff --git a/setup.cfg b/setup.cfg index 3b593d6b..de3fce30 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ test = pytest [flake8] -extend-ignore = D211,E731,D401,W503 +extend-ignore = D211,E731,D401,W503,D202 max-line-length = 100 per-file-ignores = test/*:D1 diff --git a/setup.py b/setup.py index dea5ae0f..ea12903c 100644 --- a/setup.py +++ b/setup.py @@ -13,26 +13,31 @@ ] needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv) -pytest_runner = ['pytest-runner'] if needs_pytest else [] setup_requires = [ -] + pytest_runner +] tests_require = [ 'pytest', + 'pytest_timeout', +] + +docs_require = [ + 'sphinx', + 'sphinx-rtd-theme', ] extras_require = { - 'pyuv': ['pyuv>=1.0.0'], 'test': tests_require, + 'docs': docs_require, } if platform.python_implementation() != 'PyPy': # pypy already includes an implementation of the greenlet module install_requires.append('greenlet>=3.0') -if sys.version_info < (3, 8): - install_requires.append('typing-extensions') +if sys.version_info < (3, 12): + install_requires.append('typing-extensions>=4.5') # __version__: see pynvim/_version.py diff --git a/test/conftest.py b/test/conftest.py index 4b032a24..49ed3305 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -19,7 +19,7 @@ def vim() -> Generator[pynvim.Nvim, None, None]: editor: pynvim.Nvim child_argv = os.environ.get('NVIM_CHILD_ARGV') - listen_address = os.environ.get('NVIM_LISTEN_ADDRESS') + listen_address = os.environ.get('NVIM') if child_argv is None and listen_address is None: child_argv = json.dumps([ "nvim", diff --git a/test/test_attach.py b/test/test_attach.py new file mode 100644 index 00000000..c1588ca0 --- /dev/null +++ b/test/test_attach.py @@ -0,0 +1,137 @@ +"""Tests other session_types than subprocess Nvim.""" + +import contextlib +import os.path +import socket +import subprocess +import tempfile +import time +from typing import Generator + +import pytest +import pytest_timeout # pylint: disable=unused-import # noqa + +import pynvim +from pynvim.api import Nvim + +# pylint: disable=consider-using-with +# pylint: disable=redefined-outer-name + + +xfail_on_windows = pytest.mark.xfail( + "os.name == 'nt'", reason="Broken in Windows, see #544") + + +@pytest.fixture +def tmp_socket() -> Generator[str, None, None]: + """Get a temporary UNIX socket file.""" + # see cpython#93914 + addr = tempfile.mktemp(prefix="test_python_", suffix='.sock', + dir=os.path.curdir) + try: + yield addr + finally: + if os.path.exists(addr): + with contextlib.suppress(OSError): + os.unlink(addr) + + +@xfail_on_windows +def test_connect_socket(tmp_socket: str) -> None: + """Tests UNIX socket connection.""" + p = subprocess.Popen(["nvim", "--clean", "-n", "--headless", + "--listen", tmp_socket]) + time.sleep(0.2) # wait a bit until nvim starts up + + try: + nvim: Nvim = pynvim.attach('socket', path=tmp_socket) + assert 42 == nvim.eval('42') + assert "?" == nvim.command_output('echo "?"') + finally: + with contextlib.suppress(OSError): + p.terminate() + + +def test_connect_socket_fail() -> None: + """Tests UNIX socket connection, when the sock file is not found.""" + with pytest.raises(FileNotFoundError): + pynvim.attach('socket', path='/tmp/not-exist.socket') + + +def find_free_port() -> int: + """Find a free, available port number.""" + with socket.socket() as sock: + sock.bind(('', 0)) # Bind to a free port provided by the host. + return sock.getsockname()[1] + + +def test_connect_tcp() -> None: + """Tests TCP connection.""" + address = '127.0.0.1' + port = find_free_port() + p = subprocess.Popen(["nvim", "--clean", "-n", "--headless", + "--listen", f"{address}:{port}"]) + time.sleep(0.2) # wait a bit until nvim starts up + + try: + nvim: Nvim = pynvim.attach('tcp', address=address, port=port) + assert 42 == nvim.eval('42') + assert "?" == nvim.command_output('echo "?"') + finally: + with contextlib.suppress(OSError): + p.terminate() + + +@pytest.mark.timeout(5.0) +def test_connect_tcp_no_server() -> None: + """Tests TCP socket connection that fails; connection refused.""" + port = find_free_port() + + with pytest.raises(ConnectionRefusedError): + pynvim.attach('tcp', address='127.0.0.1', port=port) + + +@xfail_on_windows +def test_connect_stdio(vim: Nvim) -> None: + """Tests stdio connection, using jobstart(..., {'rpc': v:true}).""" + + def source(vim: Nvim, code: str) -> None: + """Source a vimscript code in the embedded nvim instance.""" + fd, fname = tempfile.mkstemp() + try: + with os.fdopen(fd, 'w') as f: + f.write(code) + vim.command('source ' + fname) + finally: + os.unlink(fname) + + # A helper function for debugging that captures what pynvim writes to + # stderr (e.g. python stacktrace): used as a |on_stderr| callback + source(vim, """ + function! OutputHandler(j, lines, event_type) + if a:event_type == 'stderr' + for l:line in a:lines + echom l:line + endfor + endif + endfunction + """) + + remote_py_code = '\n'.join([ + 'import pynvim', + 'nvim = pynvim.attach("stdio")', + 'print("rplugins can write to stdout")', # tests #377 (#60) + 'nvim.api.command("let g:success = 42")', + ]) + # see :help jobstart(), *jobstart-options* |msgpack-rpc| + jobid = vim.funcs.jobstart([ + 'python', '-c', remote_py_code, + ], {'rpc': True, 'on_stderr': 'OutputHandler'}) + assert jobid > 0 + exitcode = vim.funcs.jobwait([jobid], 500)[0] + messages = vim.command_output('messages') + assert exitcode == 0, ("the python process failed, :messages =>\n\n" + + messages) + + assert 42 == vim.eval('g:success') + assert "rplugins can write to stdout" in messages diff --git a/test/test_events.py b/test/test_events.py index 2a27a89b..c978293d 100644 --- a/test/test_events.py +++ b/test/test_events.py @@ -37,20 +37,17 @@ def test_async_error(vim: Nvim) -> None: def test_broadcast(vim: Nvim) -> None: - vim.subscribe('event2') vim.command('call rpcnotify(0, "event1", 1, 2, 3)') vim.command('call rpcnotify(0, "event2", 4, 5, 6)') vim.command('call rpcnotify(0, "event2", 7, 8, 9)') event = vim.next_message() - assert event[1] == 'event2' - assert event[2] == [4, 5, 6] + assert event[1] == 'event1' + assert event[2] == [1, 2, 3] event = vim.next_message() assert event[1] == 'event2' - assert event[2] == [7, 8, 9] - vim.unsubscribe('event2') - vim.subscribe('event1') + assert event[2] == [4, 5, 6] vim.command('call rpcnotify(0, "event2", 10, 11, 12)') vim.command('call rpcnotify(0, "event1", 13, 14, 15)') msg = vim.next_message() - assert msg[1] == 'event1' - assert msg[2] == [13, 14, 15] + assert msg[1] == 'event2' + assert msg[2] == [7, 8, 9] diff --git a/test/test_host.py b/test/test_host.py index 18cff327..c404fdc3 100644 --- a/test/test_host.py +++ b/test/test_host.py @@ -56,14 +56,3 @@ def test_host_async_error(vim): assert event[1] == 'nvim_error_event' assert 'rplugin-host: Async request caused an error:\nboom\n' \ in h._on_error_event(None, 'boom') - - -def test_legacy_vim_eval(vim): - h = ScriptHost(vim) - try: - assert h.legacy_vim.eval('1') == '1' - assert h.legacy_vim.eval('v:null') is None - assert h.legacy_vim.eval('v:true') is True - assert h.legacy_vim.eval('v:false') is False - finally: - h.teardown() diff --git a/test/test_vim.py b/test/test_vim.py index 1c12e26e..8a76f5e6 100644 --- a/test/test_vim.py +++ b/test/test_vim.py @@ -1,7 +1,8 @@ +"""Tests interaction with neovim via Nvim API (with child process).""" + import os import sys import tempfile -import textwrap from pathlib import Path import pytest @@ -207,10 +208,11 @@ def test_hash(vim: Nvim) -> None: def test_python3(vim: Nvim) -> None: """Tests whether python3 host can load.""" - python3_prog = vim.command_output('echom provider#python3#Prog()') - python3_err = vim.command_output('echom provider#python3#Error()') - assert python3_prog != "", python3_err - assert python3_prog == sys.executable + rv = vim.exec_lua(''' + local prog, err = vim.provider.python.detect_by_module("neovim") + return { prog = prog, err = err }''') + assert rv['prog'] != "", rv['err'] + assert rv['prog'] == sys.executable assert sys.executable == vim.command_output( 'python3 import sys; print(sys.executable)') @@ -229,21 +231,16 @@ def test_python3_ex_eval(vim: Nvim) -> None: # because the Ex command :python will throw (wrapped with provider#python3#Call) with pytest.raises(NvimError) as excinfo: vim.command('py3= 1/0') - assert textwrap.dedent('''\ - Traceback (most recent call last): - File "", line 1, in - ZeroDivisionError: division by zero - ''').strip() in excinfo.value.args[0] + stacktrace = excinfo.value.args[0] + assert 'File "", line 1, in ' in stacktrace + assert 'ZeroDivisionError: division by zero' in stacktrace vim.command('python3 def raise_error(): raise RuntimeError("oops")') with pytest.raises(NvimError) as excinfo: vim.command_output('python3 =print("nooo", raise_error())') - assert textwrap.dedent('''\ - Traceback (most recent call last): - File "", line 1, in - File "", line 1, in raise_error - RuntimeError: oops - ''').strip() in excinfo.value.args[0] + stacktrace = excinfo.value.args[0] + assert 'File "", line 1, in raise_error' in stacktrace + assert 'RuntimeError: oops' in stacktrace assert 'nooo' not in vim.command_output(':messages') diff --git a/tox.ini b/tox.ini index a57a7909..d215e606 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,6 @@ extras = test deps = pytest-timeout # cov: pytest-cov -# pyuv: pyuv # setenv = # cov: PYTEST_ADDOPTS=--cov=. {env:PYTEST_ADDOPTS:} # passenv = PYTEST_ADDOPTS @@ -50,8 +49,8 @@ commands = [testenv:docs] deps = - Sphinx - sphinx_rtd_theme + sphinx + sphinx-rtd-theme changedir = {toxinidir}/docs commands = sphinx-build -b html -d {envtmpdir}/doctrees . {envtmpdir}/html