Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 631fd38

Browse files
1st11st1
authored
bpo-32251: Implement asyncio.BufferedProtocol. (#4755)
1 parent 0ceb717 commit 631fd38

8 files changed

Lines changed: 763 additions & 44 deletions

File tree

‎Doc/library/asyncio-protocol.rst‎

Lines changed: 71 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,16 @@ Protocol classes
333333
The base class for implementing streaming protocols (for use with
334334
e.g. TCP and SSL transports).
335335

336+
.. class:: BufferedProtocol
337+
338+
A base class for implementing streaming protocols with manual
339+
control of the receive buffer.
340+
341+
.. versionadded:: 3.7
342+
**Important:** this has been been added to asyncio in Python 3.7
343+
*on a provisional basis*! Treat it as an experimental API that
344+
might be changed or removed in Python 3.8.
345+
336346
.. class:: DatagramProtocol
337347

338348
The base class for implementing datagram protocols (for use with
@@ -428,10 +438,67 @@ and, if called, :meth:`data_received` won't be called after it.
428438

429439
State machine:
430440

431-
start -> :meth:`~BaseProtocol.connection_made`
432-
[-> :meth:`~Protocol.data_received` \*]
433-
[-> :meth:`~Protocol.eof_received` ?]
434-
-> :meth:`~BaseProtocol.connection_lost` -> end
441+
.. code-block:: none
442+
443+
start -> connection_made
444+
[-> data_received]*
445+
[-> eof_received]?
446+
-> connection_lost -> end
447+
448+
449+
Streaming protocols with manual receive buffer control
450+
------------------------------------------------------
451+
452+
.. versionadded:: 3.7
453+
**Important:** :class:`BufferedProtocol` has been been added to
454+
asyncio in Python 3.7 *on a provisional basis*! Treat it as an
455+
experimental API that might be changed or removed in Python 3.8.
456+
457+
458+
Event methods, such as :meth:`AbstractEventLoop.create_server` and
459+
:meth:`AbstractEventLoop.create_connection`, accept factories that
460+
return protocols that implement this interface.
461+
462+
The idea of BufferedProtocol is that it allows to manually allocate
463+
and control the receive buffer. Event loops can then use the buffer
464+
provided by the protocol to avoid unnecessary data copies. This
465+
can result in noticeable performance improvement for protocols that
466+
receive big amounts of data. Sophisticated protocols can allocate
467+
the buffer only once at creation time.
468+
469+
The following callbacks are called on :class:`BufferedProtocol`
470+
instances:
471+
472+
.. method:: BufferedProtocol.get_buffer()
473+
474+
Called to allocate a new receive buffer. Must return an object
475+
that implements the :ref:`buffer protocol <bufferobjects>`.
476+
477+
.. method:: BufferedProtocol.buffer_updated(nbytes)
478+
479+
Called when the buffer was updated with the received data.
480+
481+
*nbytes* is the total number of bytes that were written to the buffer.
482+
483+
.. method:: BufferedProtocol.eof_received()
484+
485+
See the documentation of the :meth:`Protocol.eof_received` method.
486+
487+
488+
:meth:`get_buffer` can be called an arbitrary number of times during
489+
a connection. However, :meth:`eof_received` is called at most once
490+
and, if called, :meth:`data_received` won't be called after it.
491+
492+
State machine:
493+
494+
.. code-block:: none
495+
496+
start -> connection_made
497+
[-> get_buffer
498+
[-> buffer_updated]?
499+
]*
500+
[-> eof_received]?
501+
-> connection_lost -> end
435502
436503
437504
Datagram protocols

‎Lib/asyncio/proactor_events.py‎

Lines changed: 117 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from . import base_events
1313
from . import constants
1414
from . import futures
15+
from . import protocols
1516
from . import sslproto
1617
from . import transports
1718
from .log import logger
@@ -91,17 +92,19 @@ def __del__(self):
9192
self.close()
9293

9394
def _fatal_error(self, exc, message='Fatal error on pipe transport'):
94-
if isinstance(exc, base_events._FATAL_ERROR_IGNORE):
95-
if self._loop.get_debug():
96-
logger.debug("%r: %s", self, message, exc_info=True)
97-
else:
98-
self._loop.call_exception_handler({
99-
'message': message,
100-
'exception': exc,
101-
'transport': self,
102-
'protocol': self._protocol,
103-
})
104-
self._force_close(exc)
95+
try:
96+
if isinstance(exc, base_events._FATAL_ERROR_IGNORE):
97+
if self._loop.get_debug():
98+
logger.debug("%r: %s", self, message, exc_info=True)
99+
else:
100+
self._loop.call_exception_handler({
101+
'message': message,
102+
'exception': exc,
103+
'transport': self,
104+
'protocol': self._protocol,
105+
})
106+
finally:
107+
self._force_close(exc)
105108

106109
def _force_close(self, exc):
107110
if self._closing:
@@ -150,6 +153,12 @@ def __init__(self, loop, sock, protocol, waiter=None,
150153
extra=None, server=None):
151154
super().__init__(loop, sock, protocol, waiter, extra, server)
152155
self._paused = False
156+
157+
if protocols._is_buffered_protocol(protocol):
158+
self._loop_reading = self._loop_reading__get_buffer
159+
else:
160+
self._loop_reading = self._loop_reading__data_received
161+
153162
self._loop.call_soon(self._loop_reading)
154163

155164
def is_reading(self):
@@ -159,6 +168,11 @@ def pause_reading(self):
159168
if self._closing or self._paused:
160169
return
161170
self._paused = True
171+
172+
if self._read_fut is not None and not self._read_fut.done():
173+
self._read_fut.cancel()
174+
self._read_fut = None
175+
162176
if self._loop.get_debug():
163177
logger.debug("%r pauses reading", self)
164178

@@ -170,11 +184,25 @@ def resume_reading(self):
170184
if self._loop.get_debug():
171185
logger.debug("%r resumes reading", self)
172186

173-
def _loop_reading(self, fut=None):
187+
def _loop_reading__on_eof(self):
188+
if self._loop.get_debug():
189+
logger.debug("%r received EOF", self)
190+
191+
try:
192+
keep_open = self._protocol.eof_received()
193+
except Exception as exc:
194+
self._fatal_error(
195+
exc, 'Fatal error: protocol.eof_received() call failed.')
196+
return
197+
198+
if not keep_open:
199+
self.close()
200+
201+
def _loop_reading__data_received(self, fut=None):
174202
if self._paused:
175203
return
176-
data = None
177204

205+
data = None
178206
try:
179207
if fut is not None:
180208
assert self._read_fut is fut or (self._read_fut is None and
@@ -197,7 +225,7 @@ def _loop_reading(self, fut=None):
197225
return
198226

199227
# reschedule a new read
200-
self._read_fut = self._loop._proactor.recv(self._sock, 4096)
228+
self._read_fut = self._loop._proactor.recv(self._sock, 32768)
201229
except ConnectionAbortedError as exc:
202230
if not self._closing:
203231
self._fatal_error(exc, 'Fatal read error on pipe transport')
@@ -216,12 +244,81 @@ def _loop_reading(self, fut=None):
216244
finally:
217245
if data:
218246
self._protocol.data_received(data)
219-
elif data is not None:
220-
if self._loop.get_debug():
221-
logger.debug("%r received EOF", self)
222-
keep_open = self._protocol.eof_received()
223-
if not keep_open:
224-
self.close()
247+
elif data == b'':
248+
self._loop_reading__on_eof()
249+
250+
def _loop_reading__get_buffer(self, fut=None):
251+
if self._paused:
252+
return
253+
254+
nbytes = None
255+
if fut is not None:
256+
assert self._read_fut is fut or (self._read_fut is None and
257+
self._closing)
258+
self._read_fut = None
259+
try:
260+
if fut.done():
261+
nbytes = fut.result()
262+
else:
263+
# the future will be replaced by next proactor.recv call
264+
fut.cancel()
265+
except ConnectionAbortedError as exc:
266+
if not self._closing:
267+
self._fatal_error(
268+
exc, 'Fatal read error on pipe transport')
269+
elif self._loop.get_debug():
270+
logger.debug("Read error on pipe transport while closing",
271+
exc_info=True)
272+
except ConnectionResetError as exc:
273+
self._force_close(exc)
274+
except OSError as exc:
275+
self._fatal_error(exc, 'Fatal read error on pipe transport')
276+
except futures.CancelledError:
277+
if not self._closing:
278+
raise
279+
280+
if nbytes is not None:
281+
if nbytes == 0:
282+
# we got end-of-file so no need to reschedule a new read
283+
self._loop_reading__on_eof()
284+
else:
285+
try:
286+
self._protocol.buffer_updated(nbytes)
287+
except Exception as exc:
288+
self._fatal_error(
289+
exc,
290+
'Fatal error: '
291+
'protocol.buffer_updated() call failed.')
292+
return
293+
294+
if self._closing or nbytes == 0:
295+
# since close() has been called we ignore any read data
296+
return
297+
298+
try:
299+
buf = self._protocol.get_buffer()
300+
except Exception as exc:
301+
self._fatal_error(
302+
exc, 'Fatal error: protocol.get_buffer() call failed.')
303+
return
304+
305+
try:
306+
# schedule a new read
307+
self._read_fut = self._loop._proactor.recv_into(self._sock, buf)
308+
self._read_fut.add_done_callback(self._loop_reading)
309+
except ConnectionAbortedError as exc:
310+
if not self._closing:
311+
self._fatal_error(exc, 'Fatal read error on pipe transport')
312+
elif self._loop.get_debug():
313+
logger.debug("Read error on pipe transport while closing",
314+
exc_info=True)
315+
except ConnectionResetError as exc:
316+
self._force_close(exc)
317+
except OSError as exc:
318+
self._fatal_error(exc, 'Fatal read error on pipe transport')
319+
except futures.CancelledError:
320+
if not self._closing:
321+
raise
225322

226323

227324
class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,

‎Lib/asyncio/protocols.py‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
__all__ = (
44
'BaseProtocol', 'Protocol', 'DatagramProtocol',
5-
'SubprocessProtocol',
5+
'SubprocessProtocol', 'BufferedProtocol',
66
)
77

88

@@ -102,6 +102,57 @@ def eof_received(self):
102102
"""
103103

104104

105+
class BufferedProtocol(BaseProtocol):
106+
"""Interface for stream protocol with manual buffer control.
107+
108+
Important: this has been been added to asyncio in Python 3.7
109+
*on a provisional basis*! Treat it as an experimental API that
110+
might be changed or removed in Python 3.8.
111+
112+
Event methods, such as `create_server` and `create_connection`,
113+
accept factories that return protocols that implement this interface.
114+
115+
The idea of BufferedProtocol is that it allows to manually allocate
116+
and control the receive buffer. Event loops can then use the buffer
117+
provided by the protocol to avoid unnecessary data copies. This
118+
can result in noticeable performance improvement for protocols that
119+
receive big amounts of data. Sophisticated protocols can allocate
120+
the buffer only once at creation time.
121+
122+
State machine of calls:
123+
124+
start -> CM [-> GB [-> BU?]]* [-> ER?] -> CL -> end
125+
126+
* CM: connection_made()
127+
* GB: get_buffer()
128+
* BU: buffer_updated()
129+
* ER: eof_received()
130+
* CL: connection_lost()
131+
"""
132+
133+
def get_buffer(self):
134+
"""Called to allocate a new receive buffer.
135+
136+
Must return an object that implements the
137+
:ref:`buffer protocol <bufferobjects>`.
138+
"""
139+
140+
def buffer_updated(self, nbytes):
141+
"""Called when the buffer was updated with the received data.
142+
143+
*nbytes* is the total number of bytes that were written to
144+
the buffer.
145+
"""
146+
147+
def eof_received(self):
148+
"""Called when the other end calls write_eof() or equivalent.
149+
150+
If this returns a false value (including None), the transport
151+
will close itself. If it returns a true value, closing the
152+
transport is up to the protocol.
153+
"""
154+
155+
105156
class DatagramProtocol(BaseProtocol):
106157
"""Interface for datagram protocol."""
107158

@@ -134,3 +185,7 @@ def pipe_connection_lost(self, fd, exc):
134185

135186
def process_exited(self):
136187
"""Called when subprocess has exited."""
188+
189+
190+
def _is_buffered_protocol(proto):
191+
return hasattr(proto, 'get_buffer') and not hasattr(proto, 'data_received')

0 commit comments

Comments
 (0)