Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit bd73e72

Browse files
drallensmithpitrou
drallensmith
authored and
pitrou
committed
bpo-5001: More-informative multiprocessing error messages (#3079)
* Make error message more informative Replace assertions in error-reporting code with more-informative version that doesn't cause confusion over where and what the error is. * Additional clarification + get travis to check * Change from SystemError to TypeError As suggested in PR comment by @pitrou, changing from SystemError; TypeError appears appropriate. * NEWS file installation; ACKS addition (will do my best to justify it by additional work) * Making current AssertionErrors in multiprocessing more informative * Blurb added re multiprocessing managers.py, queues.py cleanup * Further multiprocessing cleanup - went through pool.py * Fix two asserts in multiprocessing/util.py * Most asserts in multiprocessing more informative * Didn't save right version * Further work on multiprocessing error messages * Correct typo * Correct typo v2 * Blasted colon... serves me right for trying to work on two things at once * Simplify NEWS entry * Update 2017-08-18-17-16-38.bpo-5001.gwnthq.rst * Update 2017-08-18-17-16-38.bpo-5001.gwnthq.rst OK, never mind. * Corrected (thanks to pitrou) error messages for notify * Remove extraneous backslash in docstring.
1 parent 631fdee commit bd73e72

15 files changed

Lines changed: 120 additions & 39 deletions

‎Lib/multiprocessing/connection.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -720,7 +720,9 @@ def PipeClient(address):
720720

721721
def deliver_challenge(connection, authkey):
722722
import hmac
723-
assert isinstance(authkey, bytes)
723+
if not isinstance(authkey, bytes):
724+
raise ValueError(
725+
"Authkey must be bytes, not {0!s}".format(type(authkey)))
724726
message = os.urandom(MESSAGE_LENGTH)
725727
connection.send_bytes(CHALLENGE + message)
726728
digest = hmac.new(authkey, message, 'md5').digest()
@@ -733,7 +735,9 @@ def deliver_challenge(connection, authkey):
733735

734736
def answer_challenge(connection, authkey):
735737
import hmac
736-
assert isinstance(authkey, bytes)
738+
if not isinstance(authkey, bytes):
739+
raise ValueError(
740+
"Authkey must be bytes, not {0!s}".format(type(authkey)))
737741
message = connection.recv_bytes(256) # reject large message
738742
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
739743
message = message[len(CHALLENGE):]

‎Lib/multiprocessing/dummy/__init__.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,10 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs={}):
4141
self._parent = current_process()
4242

4343
def start(self):
44-
assert self._parent is current_process()
44+
if self._parent is not current_process():
45+
raise RuntimeError(
46+
"Parent is {0!r} but current_process is {1!r}".format(
47+
self._parent, current_process()))
4548
self._start_called = True
4649
if hasattr(self._parent, '_children'):
4750
self._parent._children[self] = None

‎Lib/multiprocessing/forkserver.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ def sigchld_handler(*_unused):
189189

190190
if alive_r in rfds:
191191
# EOF because no more client processes left
192-
assert os.read(alive_r, 1) == b''
192+
assert os.read(alive_r, 1) == b'', "Not at EOF?"
193193
raise SystemExit
194194

195195
if sig_r in rfds:
@@ -208,7 +208,10 @@ def sigchld_handler(*_unused):
208208
if os.WIFSIGNALED(sts):
209209
returncode = -os.WTERMSIG(sts)
210210
else:
211-
assert os.WIFEXITED(sts)
211+
if not os.WIFEXITED(sts):
212+
raise AssertionError(
213+
"Child {0:n} status is {1:n}".format(
214+
pid,sts))
212215
returncode = os.WEXITSTATUS(sts)
213216
# Send exit code to client process
214217
try:
@@ -227,7 +230,10 @@ def sigchld_handler(*_unused):
227230
with listener.accept()[0] as s:
228231
# Receive fds from client
229232
fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
230-
assert len(fds) <= MAXFDS_TO_SEND
233+
if len(fds) > MAXFDS_TO_SEND:
234+
raise RuntimeError(
235+
"Too many ({0:n}) fds to send".format(
236+
len(fds)))
231237
child_r, child_w, *fds = fds
232238
s.close()
233239
pid = os.fork()

‎Lib/multiprocessing/heap.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,10 @@ def free(self, block):
211211
# synchronously sometimes later from malloc() or free(), by calling
212212
# _free_pending_blocks() (appending and retrieving from a list is not
213213
# strictly thread-safe but under cPython it's atomic thanks to the GIL).
214-
assert os.getpid() == self._lastpid
214+
if os.getpid() != self._lastpid:
215+
raise ValueError(
216+
"My pid ({0:n}) is not last pid {1:n}".format(
217+
os.getpid(),self._lastpid))
215218
if not self._lock.acquire(False):
216219
# can't acquire the lock right now, add the block to the list of
217220
# pending blocks to free
@@ -227,7 +230,10 @@ def free(self, block):
227230

228231
def malloc(self, size):
229232
# return a block of right size (possibly rounded up)
230-
assert 0 <= size < sys.maxsize
233+
if size < 0:
234+
raise ValueError("Size {0:n} out of range".format(size))
235+
if sys.maxsize <= size:
236+
raise OverflowError("Size {0:n} too large".format(size))
231237
if os.getpid() != self._lastpid:
232238
self.__init__() # reinitialize after fork
233239
with self._lock:
@@ -250,7 +256,10 @@ class BufferWrapper(object):
250256
_heap = Heap()
251257

252258
def __init__(self, size):
253-
assert 0 <= size < sys.maxsize
259+
if size < 0:
260+
raise ValueError("Size {0:n} out of range".format(size))
261+
if sys.maxsize <= size:
262+
raise OverflowError("Size {0:n} too large".format(size))
254263
block = BufferWrapper._heap.malloc(size)
255264
self._state = (block, size)
256265
util.Finalize(self, BufferWrapper._heap.free, args=(block,))

‎Lib/multiprocessing/managers.py‎

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from traceback import format_exc
2424

2525
from . import connection
26-
from .context import reduction, get_spawning_popen
26+
from .context import reduction, get_spawning_popen, ProcessError
2727
from . import pool
2828
from . import process
2929
from . import util
@@ -133,7 +133,10 @@ class Server(object):
133133
'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref']
134134

135135
def __init__(self, registry, address, authkey, serializer):
136-
assert isinstance(authkey, bytes)
136+
if not isinstance(authkey, bytes):
137+
raise TypeError(
138+
"Authkey {0!r} is type {1!s}, not bytes".format(
139+
authkey, type(authkey)))
137140
self.registry = registry
138141
self.authkey = process.AuthenticationString(authkey)
139142
Listener, Client = listener_client[serializer]
@@ -163,7 +166,7 @@ def serve_forever(self):
163166
except (KeyboardInterrupt, SystemExit):
164167
pass
165168
finally:
166-
if sys.stdout != sys.__stdout__:
169+
if sys.stdout != sys.__stdout__: # what about stderr?
167170
util.debug('resetting stdout, stderr')
168171
sys.stdout = sys.__stdout__
169172
sys.stderr = sys.__stderr__
@@ -316,6 +319,7 @@ def debug_info(self, c):
316319
'''
317320
Return some info --- useful to spot problems with refcounting
318321
'''
322+
# Perhaps include debug info about 'c'?
319323
with self.mutex:
320324
result = []
321325
keys = list(self.id_to_refcount.keys())
@@ -356,15 +360,20 @@ def create(self, c, typeid, *args, **kwds):
356360
self.registry[typeid]
357361

358362
if callable is None:
359-
assert len(args) == 1 and not kwds
363+
if kwds or (len(args) != 1):
364+
raise ValueError(
365+
"Without callable, must have one non-keyword argument")
360366
obj = args[0]
361367
else:
362368
obj = callable(*args, **kwds)
363369

364370
if exposed is None:
365371
exposed = public_methods(obj)
366372
if method_to_typeid is not None:
367-
assert type(method_to_typeid) is dict
373+
if not isinstance(method_to_typeid, dict):
374+
raise TypeError(
375+
"Method_to_typeid {0!r}: type {1!s}, not dict".format(
376+
method_to_typeid, type(method_to_typeid)))
368377
exposed = list(exposed) + list(method_to_typeid)
369378

370379
ident = '%x' % id(obj) # convert to string because xmlrpclib
@@ -417,7 +426,11 @@ def decref(self, c, ident):
417426
return
418427

419428
with self.mutex:
420-
assert self.id_to_refcount[ident] >= 1
429+
if self.id_to_refcount[ident] <= 0:
430+
raise AssertionError(
431+
"Id {0!s} ({1!r}) has refcount {2:n}, not 1+".format(
432+
ident, self.id_to_obj[ident],
433+
self.id_to_refcount[ident]))
421434
self.id_to_refcount[ident] -= 1
422435
if self.id_to_refcount[ident] == 0:
423436
del self.id_to_refcount[ident]
@@ -480,7 +493,14 @@ def get_server(self):
480493
'''
481494
Return server object with serve_forever() method and address attribute
482495
'''
483-
assert self._state.value == State.INITIAL
496+
if self._state.value != State.INITIAL:
497+
if self._state.value == State.STARTED:
498+
raise ProcessError("Already started server")
499+
elif self._state.value == State.SHUTDOWN:
500+
raise ProcessError("Manager has shut down")
501+
else:
502+
raise ProcessError(
503+
"Unknown state {!r}".format(self._state.value))
484504
return Server(self._registry, self._address,
485505
self._authkey, self._serializer)
486506

@@ -497,7 +517,14 @@ def start(self, initializer=None, initargs=()):
497517
'''
498518
Spawn a server process for this manager object
499519
'''
500-
assert self._state.value == State.INITIAL
520+
if self._state.value != State.INITIAL:
521+
if self._state.value == State.STARTED:
522+
raise ProcessError("Already started server")
523+
elif self._state.value == State.SHUTDOWN:
524+
raise ProcessError("Manager has shut down")
525+
else:
526+
raise ProcessError(
527+
"Unknown state {!r}".format(self._state.value))
501528

502529
if initializer is not None and not callable(initializer):
503530
raise TypeError('initializer must be a callable')
@@ -593,7 +620,14 @@ def _number_of_objects(self):
593620
def __enter__(self):
594621
if self._state.value == State.INITIAL:
595622
self.start()
596-
assert self._state.value == State.STARTED
623+
if self._state.value != State.STARTED:
624+
if self._state.value == State.INITIAL:
625+
raise ProcessError("Unable to start server")
626+
elif self._state.value == State.SHUTDOWN:
627+
raise ProcessError("Manager has shut down")
628+
else:
629+
raise ProcessError(
630+
"Unknown state {!r}".format(self._state.value))
597631
return self
598632

599633
def __exit__(self, exc_type, exc_val, exc_tb):
@@ -653,7 +687,7 @@ def register(cls, typeid, callable=None, proxytype=None, exposed=None,
653687
getattr(proxytype, '_method_to_typeid_', None)
654688

655689
if method_to_typeid:
656-
for key, value in list(method_to_typeid.items()):
690+
for key, value in list(method_to_typeid.items()): # isinstance?
657691
assert type(key) is str, '%r is not a string' % key
658692
assert type(value) is str, '%r is not a string' % value
659693

‎Lib/multiprocessing/pool.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,9 @@ def __repr__(self):
9292

9393
def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None,
9494
wrap_exception=False):
95-
assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0)
95+
if (maxtasks is not None) and not (isinstance(maxtasks, int)
96+
and maxtasks >= 1):
97+
raise AssertionError("Maxtasks {!r} is not valid".format(maxtasks))
9698
put = outqueue.put
9799
get = inqueue.get
98100
if hasattr(inqueue, '_writer'):
@@ -254,8 +256,8 @@ def _setup_queues(self):
254256
def apply(self, func, args=(), kwds={}):
255257
'''
256258
Equivalent of `func(*args, **kwds)`.
259+
Pool must be running.
257260
'''
258-
assert self._state == RUN
259261
return self.apply_async(func, args, kwds).get()
260262

261263
def map(self, func, iterable, chunksize=None):
@@ -307,6 +309,10 @@ def imap(self, func, iterable, chunksize=1):
307309
))
308310
return result
309311
else:
312+
if chunksize < 1:
313+
raise ValueError(
314+
"Chunksize must be 1+, not {0:n}".format(
315+
chunksize))
310316
assert chunksize > 1
311317
task_batches = Pool._get_tasks(func, iterable, chunksize)
312318
result = IMapIterator(self._cache)
@@ -334,7 +340,9 @@ def imap_unordered(self, func, iterable, chunksize=1):
334340
))
335341
return result
336342
else:
337-
assert chunksize > 1
343+
if chunksize < 1:
344+
raise ValueError(
345+
"Chunksize must be 1+, not {0!r}".format(chunksize))
338346
task_batches = Pool._get_tasks(func, iterable, chunksize)
339347
result = IMapUnorderedIterator(self._cache)
340348
self._taskqueue.put(
@@ -466,7 +474,7 @@ def _handle_results(outqueue, get, cache):
466474
return
467475

468476
if thread._state:
469-
assert thread._state == TERMINATE
477+
assert thread._state == TERMINATE, "Thread not in TERMINATE"
470478
util.debug('result handler found thread._state=TERMINATE')
471479
break
472480

@@ -542,7 +550,10 @@ def terminate(self):
542550

543551
def join(self):
544552
util.debug('joining pool')
545-
assert self._state in (CLOSE, TERMINATE)
553+
if self._state == RUN:
554+
raise ValueError("Pool is still running")
555+
elif self._state not in (CLOSE, TERMINATE):
556+
raise ValueError("In unknown state")
546557
self._worker_handler.join()
547558
self._task_handler.join()
548559
self._result_handler.join()
@@ -570,7 +581,9 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool,
570581
util.debug('helping task handler/workers to finish')
571582
cls._help_stuff_finish(inqueue, task_handler, len(pool))
572583

573-
assert result_handler.is_alive() or len(cache) == 0
584+
if (not result_handler.is_alive()) and (len(cache) != 0):
585+
raise AssertionError(
586+
"Cannot have cache with result_hander not alive")
574587

575588
result_handler._state = TERMINATE
576589
outqueue.put(None) # sentinel
@@ -628,7 +641,8 @@ def ready(self):
628641
return self._event.is_set()
629642

630643
def successful(self):
631-
assert self.ready()
644+
if not self.ready():
645+
raise ValueError("{0!r} not ready".format(self))
632646
return self._success
633647

634648
def wait(self, timeout=None):

‎Lib/multiprocessing/popen_fork.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ def poll(self, flag=os.WNOHANG):
3535
if os.WIFSIGNALED(sts):
3636
self.returncode = -os.WTERMSIG(sts)
3737
else:
38-
assert os.WIFEXITED(sts)
38+
assert os.WIFEXITED(sts), "Status is {:n}".format(sts)
3939
self.returncode = os.WEXITSTATUS(sts)
4040
return self.returncode
4141

‎Lib/multiprocessing/queues.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ def _after_fork(self):
7878
self._poll = self._reader.poll
7979

8080
def put(self, obj, block=True, timeout=None):
81-
assert not self._closed
81+
assert not self._closed, "Queue {0!r} has been closed".format(self)
8282
if not self._sem.acquire(block, timeout):
8383
raise Full
8484

@@ -140,7 +140,7 @@ def close(self):
140140

141141
def join_thread(self):
142142
debug('Queue.join_thread()')
143-
assert self._closed
143+
assert self._closed, "Queue {0!r} not closed".format(self)
144144
if self._jointhread:
145145
self._jointhread()
146146

@@ -281,7 +281,7 @@ def __setstate__(self, state):
281281
self._cond, self._unfinished_tasks = state[-2:]
282282

283283
def put(self, obj, block=True, timeout=None):
284-
assert not self._closed
284+
assert not self._closed, "Queue {0!r} is closed".format(self)
285285
if not self._sem.acquire(block, timeout):
286286
raise Full
287287

‎Lib/multiprocessing/reduction.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,10 @@ def recvfds(sock, size):
165165
if len(cmsg_data) % a.itemsize != 0:
166166
raise ValueError
167167
a.frombytes(cmsg_data)
168-
assert len(a) % 256 == msg[0]
168+
if len(a) % 256 != msg[0]:
169+
raise AssertionError(
170+
"Len is {0:n} but msg[0] is {1!r}".format(
171+
len(a), msg[0]))
169172
return list(a)
170173
except (ValueError, IndexError):
171174
pass

‎Lib/multiprocessing/resource_sharer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def _afterfork(self):
125125

126126
def _start(self):
127127
from .connection import Listener
128-
assert self._listener is None
128+
assert self._listener is None, "Already have Listener"
129129
util.debug('starting listener and thread for sending handles')
130130
self._listener = Listener(authkey=process.current_process().authkey)
131131
self._address = self._listener.address

0 commit comments

Comments
 (0)