Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 96d37db

Browse files
ltfishpitrou
ltfish
authored and
pitrou
committed
bpo-35615: Fix crashes when copying a Weak{Key,Value}Dictionary. (GH-11384)
Protect dict iterations by wrapping them with _IterationGuard in the following methods: - WeakValueDictionary.copy() - WeakValueDictionary.__deepcopy__() - WeakKeyDictionary.copy() - WeakKeyDictionary.__deepcopy__()
1 parent df8d2cd commit 96d37db

3 files changed

Lines changed: 105 additions & 16 deletions

File tree

‎Lib/test/test_weakref.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import copy
99
import threading
1010
import time
11+
import random
1112

1213
from test import support
1314
from test.support import script_helper
@@ -1688,6 +1689,87 @@ def test_threaded_weak_valued_consistency(self):
16881689
self.assertEqual(len(d), 1)
16891690
o = None # lose ref
16901691

1692+
def check_threaded_weak_dict_copy(self, type_, deepcopy):
1693+
# `type_` should be either WeakKeyDictionary or WeakValueDictionary.
1694+
# `deepcopy` should be either True or False.
1695+
exc = []
1696+
1697+
class DummyKey:
1698+
def __init__(self, ctr):
1699+
self.ctr = ctr
1700+
1701+
class DummyValue:
1702+
def __init__(self, ctr):
1703+
self.ctr = ctr
1704+
1705+
def dict_copy(d, exc):
1706+
try:
1707+
if deepcopy is True:
1708+
_ = copy.deepcopy(d)
1709+
else:
1710+
_ = d.copy()
1711+
except Exception as ex:
1712+
exc.append(ex)
1713+
1714+
def pop_and_collect(lst):
1715+
gc_ctr = 0
1716+
while lst:
1717+
i = random.randint(0, len(lst) - 1)
1718+
gc_ctr += 1
1719+
lst.pop(i)
1720+
if gc_ctr % 10000 == 0:
1721+
gc.collect() # just in case
1722+
1723+
self.assertIn(type_, (weakref.WeakKeyDictionary, weakref.WeakValueDictionary))
1724+
1725+
d = type_()
1726+
keys = []
1727+
values = []
1728+
# Initialize d with many entries
1729+
for i in range(70000):
1730+
k, v = DummyKey(i), DummyValue(i)
1731+
keys.append(k)
1732+
values.append(v)
1733+
d[k] = v
1734+
del k
1735+
del v
1736+
1737+
t_copy = threading.Thread(target=dict_copy, args=(d, exc,))
1738+
if type_ is weakref.WeakKeyDictionary:
1739+
t_collect = threading.Thread(target=pop_and_collect, args=(keys,))
1740+
else: # weakref.WeakValueDictionary
1741+
t_collect = threading.Thread(target=pop_and_collect, args=(values,))
1742+
1743+
t_copy.start()
1744+
t_collect.start()
1745+
1746+
t_copy.join()
1747+
t_collect.join()
1748+
1749+
# Test exceptions
1750+
if exc:
1751+
raise exc[0]
1752+
1753+
def test_threaded_weak_key_dict_copy(self):
1754+
# Issue #35615: Weakref keys or values getting GC'ed during dict
1755+
# copying should not result in a crash.
1756+
self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False)
1757+
1758+
def test_threaded_weak_key_dict_deepcopy(self):
1759+
# Issue #35615: Weakref keys or values getting GC'ed during dict
1760+
# copying should not result in a crash.
1761+
self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True)
1762+
1763+
def test_threaded_weak_value_dict_copy(self):
1764+
# Issue #35615: Weakref keys or values getting GC'ed during dict
1765+
# copying should not result in a crash.
1766+
self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False)
1767+
1768+
def test_threaded_weak_value_dict_deepcopy(self):
1769+
# Issue #35615: Weakref keys or values getting GC'ed during dict
1770+
# copying should not result in a crash.
1771+
self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, True)
1772+
16911773

16921774
from test import mapping_tests
16931775

‎Lib/weakref.py‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -171,10 +171,11 @@ def copy(self):
171171
if self._pending_removals:
172172
self._commit_removals()
173173
new = WeakValueDictionary()
174-
for key, wr in self.data.items():
175-
o = wr()
176-
if o is not None:
177-
new[key] = o
174+
with _IterationGuard(self):
175+
for key, wr in self.data.items():
176+
o = wr()
177+
if o is not None:
178+
new[key] = o
178179
return new
179180

180181
__copy__ = copy
@@ -184,10 +185,11 @@ def __deepcopy__(self, memo):
184185
if self._pending_removals:
185186
self._commit_removals()
186187
new = self.__class__()
187-
for key, wr in self.data.items():
188-
o = wr()
189-
if o is not None:
190-
new[deepcopy(key, memo)] = o
188+
with _IterationGuard(self):
189+
for key, wr in self.data.items():
190+
o = wr()
191+
if o is not None:
192+
new[deepcopy(key, memo)] = o
191193
return new
192194

193195
def get(self, key, default=None):
@@ -408,21 +410,23 @@ def __setitem__(self, key, value):
408410

409411
def copy(self):
410412
new = WeakKeyDictionary()
411-
for key, value in self.data.items():
412-
o = key()
413-
if o is not None:
414-
new[o] = value
413+
with _IterationGuard(self):
414+
for key, value in self.data.items():
415+
o = key()
416+
if o is not None:
417+
new[o] = value
415418
return new
416419

417420
__copy__ = copy
418421

419422
def __deepcopy__(self, memo):
420423
from copy import deepcopy
421424
new = self.__class__()
422-
for key, value in self.data.items():
423-
o = key()
424-
if o is not None:
425-
new[o] = deepcopy(value, memo)
425+
with _IterationGuard(self):
426+
for key, value in self.data.items():
427+
o = key()
428+
if o is not None:
429+
new[o] = deepcopy(value, memo)
426430
return new
427431

428432
def get(self, key, default=None):

‎Misc/NEWS.d/next/Library/2018-12-30-20-00-05.bpo-35615.Uz1SVh.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
:mod:`weakref`: Fix a RuntimeError when copying a WeakKeyDictionary or a
2+
WeakValueDictionary, due to some keys or values disappearing while
3+
iterating.

0 commit comments

Comments
 (0)