From 9b707556e04ba81b6f52607cf27565d4d395def2 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Wed, 18 May 2022 14:10:51 -0400 Subject: [PATCH 1/9] Acquire strong references after PyDict_Next --- Lib/test/pickletester.py | 39 +++++++++++++++++++++++++++++++++++++++ Modules/_pickle.c | 32 ++++++++++++++++++++++++-------- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index d0ea7d0e55e7d5..6fc18996dd2e90 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3035,6 +3035,45 @@ def check_array(arr): # 2-D, non-contiguous check_array(arr[::2]) + def test_evil_class_mutating_dict(self): + from random import getrandbits + + global Bad + class Bad: + def __eq__(self, other): + if not ENABLED: + return False + return getrandbits(4) == 0 + def __hash__(self): + return getrandbits(1) + def __reduce__(self): + break_things() + return (Bad, (), ()) + def __setstate__(self, *args): + break_things() + def __del__(self): + break_things() + def __getattr__(self): + break_things() + + def break_things(): + if ENABLED and getrandbits(6) == 0: + collection.clear() + + for proto in protocols: + for _ in range(20): + ENABLED = False + collection = {Bad(): Bad() for _ in range(50)} + for bad in collection: + bad.bad = bad + bad.collection = collection + ENABLED = True + try: + self.loads(self.dumps(collection, proto)) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) + class BigmemPickleTests: diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0adfa4103cc0d0..95add798d12d6a 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -3259,10 +3259,16 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) /* Special-case len(d) == 1 to save space. */ if (dict_size == 1) { PyDict_Next(obj, &ppos, &key, &value); - if (save(self, key, 0) < 0) - return -1; - if (save(self, value, 0) < 0) - return -1; + Py_INCREF(key); + Py_INCREF(value); + if (save(self, key, 0) < 0) { + goto error; + } + if (save(self, value, 0) < 0) { + goto error; + } + Py_CLEAR(key); + Py_CLEAR(value); if (_Pickler_Write(self, &setitem_op, 1) < 0) return -1; return 0; @@ -3274,10 +3280,16 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; while (PyDict_Next(obj, &ppos, &key, &value)) { - if (save(self, key, 0) < 0) - return -1; - if (save(self, value, 0) < 0) - return -1; + Py_INCREF(key); + Py_INCREF(value); + if (save(self, key, 0) < 0) { + goto error; + } + if (save(self, value, 0) < 0) { + goto error; + } + Py_CLEAR(key); + Py_CLEAR(value); if (++i == BATCHSIZE) break; } @@ -3292,6 +3304,10 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) } while (i == BATCHSIZE); return 0; +error: + Py_XDECREF(key); + Py_XDECREF(value); + return -1; } static int From 7af44c4fac14d6d7ed487b21d856ddafb66acbb9 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Wed, 18 May 2022 14:28:05 -0400 Subject: [PATCH 2/9] simplify test --- Lib/test/pickletester.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 6fc18996dd2e90..66e897ca732013 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3041,35 +3041,25 @@ def test_evil_class_mutating_dict(self): global Bad class Bad: def __eq__(self, other): - if not ENABLED: - return False - return getrandbits(4) == 0 + return ENABLED def __hash__(self): - return getrandbits(1) + return 42 def __reduce__(self): - break_things() - return (Bad, (), ()) - def __setstate__(self, *args): - break_things() - def __del__(self): - break_things() - def __getattr__(self): - break_things() - - def break_things(): - if ENABLED and getrandbits(6) == 0: - collection.clear() + if getrandbits(6) == 0: + collection.clear() + return (Bad, ()) for proto in protocols: for _ in range(20): ENABLED = False - collection = {Bad(): Bad() for _ in range(50)} + collection = {Bad(): Bad() for _ in range(20)} for bad in collection: bad.bad = bad bad.collection = collection ENABLED = True try: - self.loads(self.dumps(collection, proto)) + data = self.dumps(collection, proto) + self.loads(data) except RuntimeError as e: expected = "changed size during iteration" self.assertIn(expected, str(e)) From 802a22071aa11bb424dc013e02d44759e06a1eac Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 18 May 2022 18:34:48 +0000 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst new file mode 100644 index 00000000000000..72519ab4159126 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst @@ -0,0 +1 @@ +Fixed a crash when pickling objects that mutate collections during ``__reduce__``. From a2a8e35118ca472a12b2b68dcc51fce5f200d920 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Thu, 9 Jun 2022 23:23:06 -0400 Subject: [PATCH 4/9] INCREF before save() in batch_list_exact --- Lib/test/pickletester.py | 16 ++++++++++++++++ Modules/_pickle.c | 10 ++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 66e897ca732013..4261f1763afa73 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3064,6 +3064,22 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) + def test_evil_class_mutating_list(self): + if not hasattr(self, "pickler"): + raise self.skipTest(f"{type(self)} has no associated pickler type") + + global P + class P(self.pickler): + def persistent_id(self, obj): + if obj is a[0]: + a.clear() + return None + + for proto in protocols: + a = [[[[]]]] + with self.assertRaises(IndexError): + P(io.BytesIO(), proto).dump(a) + class BigmemPickleTests: diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 95add798d12d6a..a282c9c3e18e1b 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -3006,7 +3006,10 @@ batch_list_exact(PicklerObject *self, PyObject *obj) if (PyList_GET_SIZE(obj) == 1) { item = PyList_GET_ITEM(obj, 0); - if (save(self, item, 0) < 0) + Py_INCREF(item); + int err = save(self, item, 0); + Py_DECREF(item); + if (err < 0) return -1; if (_Pickler_Write(self, &append_op, 1) < 0) return -1; @@ -3021,7 +3024,10 @@ batch_list_exact(PicklerObject *self, PyObject *obj) return -1; while (total < PyList_GET_SIZE(obj)) { item = PyList_GET_ITEM(obj, total); - if (save(self, item, 0) < 0) + Py_INCREF(item); + int err = save(self, item, 0); + Py_DECREF(item); + if (err < 0) return -1; total++; if (++this_batch == BATCHSIZE) From e48b31dbf4647cf15c1e439fb036590d9e69741a Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Fri, 10 Jun 2022 00:29:59 -0400 Subject: [PATCH 5/9] Add INCREF/DECREF for save_set --- Modules/_pickle.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a282c9c3e18e1b..753ac2fe892a1b 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -3431,7 +3431,10 @@ save_set(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; while (_PySet_NextEntry(obj, &ppos, &item, &hash)) { - if (save(self, item, 0) < 0) + Py_INCREF(item); + int err = save(self, item, 0); + Py_CLEAR(item); + if (err < 0) return -1; if (++i == BATCHSIZE) break; From 7b8f697d68d1110c6ccffb3dcaafe4c9e5b7e873 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Fri, 10 Jun 2022 00:39:49 -0400 Subject: [PATCH 6/9] More tests --- Lib/test/pickletester.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 4261f1763afa73..886545e25b0dd0 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3064,21 +3064,34 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) - def test_evil_class_mutating_list(self): + def test_evil_class_mutating_container(self): if not hasattr(self, "pickler"): raise self.skipTest(f"{type(self)} has no associated pickler type") - global P - class P(self.pickler): - def persistent_id(self, obj): - if obj is a[0]: - a.clear() - return None + global Clearer + class Clearer: + pass - for proto in protocols: - a = [[[[]]]] - with self.assertRaises(IndexError): + def check(a): + class P(self.pickler): + def persistent_id(self, obj): + if isinstance(obj, Clearer): + a.clear() + return None + try: P(io.BytesIO(), proto).dump(a) + except RuntimeError: + # Don't care if this raises, just don't segfault. + pass + + for proto in protocols: + check([Clearer()]) + check([Clearer(), Clearer()]) + check([Clearer(), Clearer()]) + check({Clearer()}) + check({Clearer(): 1}) + check({Clearer(): 1, Clearer(): 2}) + check({1: Clearer(), 2: Clearer()}) class BigmemPickleTests: From 7133c2ce72c782d5204ffc1ac17d2b0a50b921cf Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Fri, 10 Jun 2022 00:42:47 -0400 Subject: [PATCH 7/9] Check non-singleton set --- Lib/test/pickletester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 886545e25b0dd0..a6d0c2f7652bc0 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3087,8 +3087,8 @@ def persistent_id(self, obj): for proto in protocols: check([Clearer()]) check([Clearer(), Clearer()]) - check([Clearer(), Clearer()]) check({Clearer()}) + check({Clearer(), Clearer()}) check({Clearer(): 1}) check({Clearer(): 1, Clearer(): 2}) check({1: Clearer(), 2: Clearer()}) From a939d16cd2aa5a5d72a61416e3f91515e106e51d Mon Sep 17 00:00:00 2001 From: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Date: Fri, 10 Jun 2022 12:56:42 -0400 Subject: [PATCH 8/9] Update Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst --- .../2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst b/Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst index 72519ab4159126..cd5d7b3214ea43 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-05-18-18-34-45.gh-issue-92930.kpYPOb.rst @@ -1 +1 @@ -Fixed a crash when pickling objects that mutate collections during ``__reduce__``. +Fixed a crash in ``_pickle.c`` from mutating collections during ``__reduce__`` or ``persistent_id``. From 370c44e650f01b2138e32590be7cbc271793b140 Mon Sep 17 00:00:00 2001 From: sweeneyde Date: Fri, 10 Jun 2022 21:14:02 -0400 Subject: [PATCH 9/9] clean up test case, better name --- Lib/test/pickletester.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index a6d0c2f7652bc0..21419e11c87497 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3036,6 +3036,7 @@ def check_array(arr): check_array(arr[::2]) def test_evil_class_mutating_dict(self): + # https://github.com/python/cpython/issues/92930 from random import getrandbits global Bad @@ -3064,7 +3065,8 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) - def test_evil_class_mutating_container(self): + def test_evil_pickler_mutating_collection(self): + # https://github.com/python/cpython/issues/92930 if not hasattr(self, "pickler"): raise self.skipTest(f"{type(self)} has no associated pickler type") @@ -3072,17 +3074,18 @@ def test_evil_class_mutating_container(self): class Clearer: pass - def check(a): - class P(self.pickler): + def check(collection): + class EvilPickler(self.pickler): def persistent_id(self, obj): if isinstance(obj, Clearer): - a.clear() + collection.clear() return None + pickler = EvilPickler(io.BytesIO(), proto) try: - P(io.BytesIO(), proto).dump(a) - except RuntimeError: - # Don't care if this raises, just don't segfault. - pass + pickler.dump(collection) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) for proto in protocols: check([Clearer()])