Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5438ed1

Browse files
committed
Issue #4892: multiprocessing Connections can now be transferred over multiprocessing Connections.
Patch by Richard Oudkerk (sbt).
1 parent 9f478c0 commit 5438ed1

8 files changed

Lines changed: 283 additions & 193 deletions

File tree

Doc/library/multiprocessing.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -832,6 +832,10 @@ Connection objects are usually created using :func:`Pipe` -- see also
832832
raised and the complete message is available as ``e.args[0]`` where ``e``
833833
is the exception instance.
834834

835+
.. versionchanged:: 3.3
836+
Connection objects themselves can now be transferred between processes
837+
using :meth:`Connection.send` and :meth:`Connection.recv`.
838+
835839

836840
For example:
837841

Lib/multiprocessing/__init__.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,9 @@ def allow_connection_pickling():
161161
'''
162162
Install support for sending connections and sockets between processes
163163
'''
164-
from multiprocessing import reduction
164+
# This is undocumented. In previous versions of multiprocessing
165+
# its only effect was to make socket objects inheritable on Windows.
166+
import multiprocessing.connection
165167

166168
#
167169
# Definitions depending on native semaphores

Lib/multiprocessing/connection.py

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
from multiprocessing import current_process, AuthenticationError, BufferTooShort
5151
from multiprocessing.util import (
5252
get_temp_dir, Finalize, sub_debug, debug, _eintr_retry)
53+
from multiprocessing.forking import ForkingPickler
5354
try:
5455
import _winapi
5556
from _winapi import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE
@@ -227,8 +228,9 @@ def send(self, obj):
227228
"""Send a (picklable) object"""
228229
self._check_closed()
229230
self._check_writable()
230-
buf = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
231-
self._send_bytes(memoryview(buf))
231+
buf = io.BytesIO()
232+
ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
233+
self._send_bytes(buf.getbuffer())
232234

233235
def recv_bytes(self, maxlength=None):
234236
"""
@@ -880,3 +882,21 @@ def wait(object_list, timeout=None):
880882
raise
881883
if timeout is not None:
882884
timeout = deadline - time.time()
885+
886+
#
887+
# Make connection and socket objects sharable if possible
888+
#
889+
890+
if sys.platform == 'win32':
891+
from . import reduction
892+
ForkingPickler.register(socket.socket, reduction.reduce_socket)
893+
ForkingPickler.register(Connection, reduction.reduce_connection)
894+
ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
895+
else:
896+
try:
897+
from . import reduction
898+
except ImportError:
899+
pass
900+
else:
901+
ForkingPickler.register(socket.socket, reduction.reduce_socket)
902+
ForkingPickler.register(Connection, reduction.reduce_connection)

Lib/multiprocessing/forking.py

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -407,25 +407,6 @@ def get_preparation_data(name):
407407

408408
return d
409409

410-
#
411-
# Make (Pipe)Connection picklable
412-
#
413-
414-
# Late import because of circular import
415-
from .connection import Connection, PipeConnection
416-
417-
def reduce_connection(conn):
418-
if not Popen.thread_is_spawning():
419-
raise RuntimeError(
420-
'By default %s objects can only be shared between processes\n'
421-
'using inheritance' % type(conn).__name__
422-
)
423-
return type(conn), (Popen.duplicate_for_child(conn.fileno()),
424-
conn.readable, conn.writable)
425-
426-
ForkingPickler.register(Connection, reduce_connection)
427-
ForkingPickler.register(PipeConnection, reduce_connection)
428-
429410
#
430411
# Prepare current process
431412
#

0 commit comments

Comments
 (0)