Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 949d47d

Browse files
amauryfaamauryfa
committed
Issue #3125: Remove copy_reg in multiprocessing and replace it with
ForkingPickler.register() to resolve conflict with ctypes.
1 parent 3ad8910 commit 949d47d

5 files changed

Lines changed: 66 additions & 58 deletions

File tree

‎Lib/multiprocessing/forking.py‎

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
from multiprocessing import util, process
1414

15-
__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close']
15+
__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close', 'ForkingPickler']
1616

1717
#
1818
# Check that the current thread is spawning a child process
@@ -25,6 +25,50 @@ def assert_spawning(self):
2525
' through inheritance' % type(self).__name__
2626
)
2727

28+
#
29+
# Try making some callable types picklable
30+
#
31+
32+
from pickle import _Pickler as Pickler
33+
class ForkingPickler(Pickler):
34+
dispatch = Pickler.dispatch.copy()
35+
@classmethod
36+
def register(cls, type, reduce):
37+
def dispatcher(self, obj):
38+
rv = reduce(obj)
39+
if isinstance(rv, str):
40+
self.save_global(obj, rv)
41+
else:
42+
self.save_reduce(obj=obj, *rv)
43+
cls.dispatch[type] = dispatcher
44+
45+
def _reduce_method(m):
46+
if m.__self__ is None:
47+
return getattr, (m.__class__, m.__func__.__name__)
48+
else:
49+
return getattr, (m.__self__, m.__func__.__name__)
50+
class _C:
51+
def f(self):
52+
pass
53+
ForkingPickler.register(type(_C().f), _reduce_method)
54+
55+
56+
def _reduce_method_descriptor(m):
57+
return getattr, (m.__objclass__, m.__name__)
58+
ForkingPickler.register(type(list.append), _reduce_method_descriptor)
59+
ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
60+
61+
try:
62+
from functools import partial
63+
except ImportError:
64+
pass
65+
else:
66+
def _reduce_partial(p):
67+
return _rebuild_partial, (p.func, p.args, p.keywords or {})
68+
def _rebuild_partial(func, args, keywords):
69+
return partial(func, *args, **keywords)
70+
ForkingPickler.register(partial, _reduce_partial)
71+
2872
#
2973
# Unix
3074
#
@@ -105,16 +149,18 @@ def thread_is_spawning():
105149
import _thread
106150
import msvcrt
107151
import _subprocess
108-
import copyreg
109152
import time
110153

111154
from ._multiprocessing import win32, Connection, PipeConnection
112155
from .util import Finalize
113156

114-
try:
115-
from cPickle import dump, load, HIGHEST_PROTOCOL
116-
except ImportError:
117-
from pickle import dump, load, HIGHEST_PROTOCOL
157+
#try:
158+
# from cPickle import dump, load, HIGHEST_PROTOCOL
159+
#except ImportError:
160+
from pickle import load, HIGHEST_PROTOCOL
161+
162+
def dump(obj, file, protocol=None):
163+
ForkingPickler(file, protocol).dump(obj)
118164

119165
#
120166
#
@@ -346,9 +392,8 @@ def reduce_connection(conn):
346392
return type(conn), (Popen.duplicate_for_child(conn.fileno()),
347393
conn.readable, conn.writable)
348394

349-
copyreg.pickle(Connection, reduce_connection)
350-
copyreg.pickle(PipeConnection, reduce_connection)
351-
395+
ForkingPickler.register(Connection, reduce_connection)
396+
ForkingPickler.register(PipeConnection, reduce_connection)
352397

353398
#
354399
# Prepare current process

‎Lib/multiprocessing/managers.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,12 @@
1818
import weakref
1919
import threading
2020
import array
21-
import copyreg
2221
import queue
2322

2423
from traceback import format_exc
2524
from multiprocessing import Process, current_process, active_children, Pool, util, connection
2625
from multiprocessing.process import AuthenticationString
27-
from multiprocessing.forking import exit, Popen, assert_spawning
26+
from multiprocessing.forking import exit, Popen, assert_spawning, ForkingPickler
2827
from multiprocessing.util import Finalize, info
2928

3029
try:
@@ -38,14 +37,14 @@
3837

3938
def reduce_array(a):
4039
return array.array, (a.typecode, a.tostring())
41-
copyreg.pickle(array.array, reduce_array)
40+
ForkingPickler.register(array.array, reduce_array)
4241

4342
view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
4443
if view_types[0] is not list: # only needed in Py3.0
4544
def rebuild_as_list(obj):
4645
return list, (list(obj),)
4746
for view_type in view_types:
48-
copyreg.pickle(view_type, rebuild_as_list)
47+
ForkingPickler.register(view_type, rebuild_as_list)
4948

5049
#
5150
# Type for identifying shared objects

‎Lib/multiprocessing/reduction.py‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,10 @@
1313
import sys
1414
import socket
1515
import threading
16-
import copyreg
1716

1817
import _multiprocessing
1918
from multiprocessing import current_process
20-
from multiprocessing.forking import Popen, duplicate, close
19+
from multiprocessing.forking import Popen, duplicate, close, ForkingPickler
2120
from multiprocessing.util import register_after_fork, debug, sub_debug
2221
from multiprocessing.connection import Client, Listener
2322

@@ -134,7 +133,7 @@ def rebuild_handle(pickled_data):
134133
return new_handle
135134

136135
#
137-
# Register `_multiprocessing.Connection` with `copy_reg`
136+
# Register `_multiprocessing.Connection` with `ForkingPickler`
138137
#
139138

140139
def reduce_connection(conn):
@@ -147,10 +146,10 @@ def rebuild_connection(reduced_handle, readable, writable):
147146
handle, readable=readable, writable=writable
148147
)
149148

150-
copyreg.pickle(_multiprocessing.Connection, reduce_connection)
149+
ForkingPickler.register(_multiprocessing.Connection, reduce_connection)
151150

152151
#
153-
# Register `socket.socket` with `copy_reg`
152+
# Register `socket.socket` with `ForkingPickler`
154153
#
155154

156155
def fromfd(fd, family, type_, proto=0):
@@ -169,10 +168,10 @@ def rebuild_socket(reduced_handle, family, type_, proto):
169168
close(fd)
170169
return _sock
171170

172-
copyreg.pickle(socket.socket, reduce_socket)
171+
ForkingPickler.register(socket.socket, reduce_socket)
173172

174173
#
175-
# Register `_multiprocessing.PipeConnection` with `copy_reg`
174+
# Register `_multiprocessing.PipeConnection` with `ForkingPickler`
176175
#
177176

178177
if sys.platform == 'win32':
@@ -187,4 +186,4 @@ def rebuild_pipe_connection(reduced_handle, readable, writable):
187186
handle, readable=readable, writable=writable
188187
)
189188

190-
copyreg.pickle(_multiprocessing.PipeConnection, reduce_pipe_connection)
189+
ForkingPickler.register(_multiprocessing.PipeConnection, reduce_pipe_connection)

‎Lib/multiprocessing/sharedctypes.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,9 @@
99
import sys
1010
import ctypes
1111
import weakref
12-
import copyreg
1312

1413
from multiprocessing import heap, RLock
15-
from multiprocessing.forking import assert_spawning
14+
from multiprocessing.forking import assert_spawning, ForkingPickler
1615

1716
__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
1817

@@ -124,8 +123,7 @@ def reduce_ctype(obj):
124123
def rebuild_ctype(type_, wrapper, length):
125124
if length is not None:
126125
type_ = type_ * length
127-
if sys.platform == 'win32' and type_ not in copyreg.dispatch_table:
128-
copyreg.pickle(type_, reduce_ctype)
126+
ForkingPickler.register(type_, reduce_ctype)
129127
obj = type_.from_address(wrapper.get_address())
130128
obj._wrapper = wrapper
131129
return obj

‎Lib/multiprocessing/util.py‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
import itertools
1010
import weakref
11-
import copyreg
1211
import atexit
1312
import threading # we want threading to install it's
1413
# cleanup function before multiprocessing does
@@ -302,35 +301,3 @@ def __init__(self):
302301
register_after_fork(self, lambda obj : obj.__dict__.clear())
303302
def __reduce__(self):
304303
return type(self), ()
305-
306-
#
307-
# Try making some callable types picklable
308-
#
309-
310-
def _reduce_method(m):
311-
if m.__self__ is None:
312-
return getattr, (m.__self__.__class__, m.__func__.__name__)
313-
else:
314-
return getattr, (m.__self__, m.__func__.__name__)
315-
copyreg.pickle(type(Finalize.__init__), _reduce_method)
316-
317-
def _reduce_method_descriptor(m):
318-
return getattr, (m.__objclass__, m.__name__)
319-
copyreg.pickle(type(list.append), _reduce_method_descriptor)
320-
copyreg.pickle(type(int.__add__), _reduce_method_descriptor)
321-
322-
def _reduce_builtin_function_or_method(m):
323-
return getattr, (m.__self__, m.__name__)
324-
copyreg.pickle(type(list().append), _reduce_builtin_function_or_method)
325-
copyreg.pickle(type(int().__add__), _reduce_builtin_function_or_method)
326-
327-
try:
328-
from functools import partial
329-
except ImportError:
330-
pass
331-
else:
332-
def _reduce_partial(p):
333-
return _rebuild_partial, (p.func, p.args, p.keywords or {})
334-
def _rebuild_partial(func, args, keywords):
335-
return partial(func, *args, **keywords)
336-
copyreg.pickle(partial, _reduce_partial)

0 commit comments

Comments
 (0)