Issue #22995: Instances of extension types with a state that aren't

serhiy-storchaka · serhiy-storchaka · commit f81be8aa3ff3 · 2015-12-25T21:04:29.000+02:00
subclasses of list or dict and haven't implemented any pickle-related
methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__,
or __getstate__), can no longer be pickled.  Including memoryview.
diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py
@@ -1,6 +1,7 @@
 # Copyright (C) 2001,2002 Python Software Foundation
 # csv package unit tests
 
+import copy
 import io
 import sys
 import os
@@ -9,6 +10,7 @@
 from tempfile import TemporaryFile
 import csv
 import gc
+import pickle
 from test import support
 
 class Test_Csv(unittest.TestCase):
@@ -424,6 +426,17 @@ def test_bad_dialect(self):
         self.assertRaises(TypeError, csv.reader, [], quoting = -1)
         self.assertRaises(TypeError, csv.reader, [], quoting = 100)
 
+    def test_copy(self):
+        for name in csv.list_dialects():
+            dialect = csv.get_dialect(name)
+            self.assertRaises(TypeError, copy.copy, dialect)
+
+    def test_pickle(self):
+        for name in csv.list_dialects():
+            dialect = csv.get_dialect(name)
+            for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+                self.assertRaises(TypeError, pickle.dumps, dialect, proto)
+
 class TestCsvBase(unittest.TestCase):
     def readerAssertEqual(self, input, expected_result):
         with TemporaryFile("w+", newline='') as fileobj:
diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py
@@ -11,6 +11,8 @@
 import weakref
 import array
 import io
+import copy
+import pickle
 
 
 class AbstractMemoryTests:
@@ -519,6 +521,17 @@ def test_memoryview_hex(self):
         m2 = m1[::-1]
         self.assertEqual(m2.hex(), '30' * 200000)
 
+    def test_copy(self):
+        m = memoryview(b'abc')
+        with self.assertRaises(TypeError):
+            copy.copy(m)
+
+    def test_pickle(self):
+        m = memoryview(b'abc')
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            with self.assertRaises(TypeError):
+                pickle.dumps(m, proto)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -10,6 +10,11 @@ Release date: tba
 Core and Builtins
 -----------------
 
+- Issue #22995: Instances of extension types with a state that aren't
+  subclasses of list or dict and haven't implemented any pickle-related
+  methods (__reduce__, __reduce_ex__, __getnewargs__, __getnewargs_ex__,
+  or __getstate__), can no longer be pickled.  Including memoryview.
+
 - Issue #20440: Massive replacing unsafe attribute setting code with special
   macro Py_SETREF.
 
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
@@ -3831,7 +3831,7 @@ _PyType_GetSlotNames(PyTypeObject *cls)
 }
 
 Py_LOCAL(PyObject *)
-_PyObject_GetState(PyObject *obj)
+_PyObject_GetState(PyObject *obj, int required)
 {
     PyObject *state;
     PyObject *getstate;
@@ -3846,6 +3846,13 @@ _PyObject_GetState(PyObject *obj)
         }
         PyErr_Clear();
 
+        if (required && obj->ob_type->tp_itemsize) {
+            PyErr_Format(PyExc_TypeError,
+                         "can't pickle %.200s objects",
+                         Py_TYPE(obj)->tp_name);
+            return NULL;
+        }
+
         {
             PyObject **dict;
             dict = _PyObject_GetDictPtr(obj);
@@ -3870,6 +3877,24 @@ _PyObject_GetState(PyObject *obj)
         }
 
         assert(slotnames == Py_None || PyList_Check(slotnames));
+        if (required) {
+            Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize;
+            if (obj->ob_type->tp_dictoffset)
+                basicsize += sizeof(PyObject *);
+            if (obj->ob_type->tp_weaklistoffset)
+                basicsize += sizeof(PyObject *);
+            if (slotnames != Py_None)
+                basicsize += sizeof(PyObject *) * Py_SIZE(slotnames);
+            if (obj->ob_type->tp_basicsize > basicsize) {
+                Py_DECREF(slotnames);
+                Py_DECREF(state);
+                PyErr_Format(PyExc_TypeError,
+                             "can't pickle %.200s objects",
+                             Py_TYPE(obj)->tp_name);
+                return NULL;
+            }
+        }
+
         if (slotnames != Py_None && Py_SIZE(slotnames) > 0) {
             PyObject *slots;
             Py_ssize_t slotnames_size, i;
@@ -4099,29 +4124,24 @@ reduce_newobj(PyObject *obj, int proto)
     PyObject *copyreg;
     PyObject *newobj, *newargs, *state, *listitems, *dictitems;
     PyObject *result;
+    int hasargs;
 
     if (Py_TYPE(obj)->tp_new == NULL) {
         PyErr_Format(PyExc_TypeError,
-                     "can't pickle %s objects",
+                     "can't pickle %.200s objects",
                      Py_TYPE(obj)->tp_name);
         return NULL;
     }
     if (_PyObject_GetNewArguments(obj, &args, &kwargs) < 0)
         return NULL;
 
-    if (args == NULL) {
-        args = PyTuple_New(0);
-        if (args == NULL) {
-            Py_XDECREF(kwargs);
-            return NULL;
-        }
-    }
     copyreg = import_copyreg();
     if (copyreg == NULL) {
-        Py_DECREF(args);
+        Py_XDECREF(args);
         Py_XDECREF(kwargs);
         return NULL;
     }
+    hasargs = (args != NULL);
     if (kwargs == NULL || PyDict_Size(kwargs) == 0) {
         _Py_IDENTIFIER(__newobj__);
         PyObject *cls;
@@ -4131,13 +4151,13 @@ reduce_newobj(PyObject *obj, int proto)
         newobj = _PyObject_GetAttrId(copyreg, &PyId___newobj__);
         Py_DECREF(copyreg);
         if (newobj == NULL) {
-            Py_DECREF(args);
+            Py_XDECREF(args);
             return NULL;
         }
-        n = PyTuple_GET_SIZE(args);
+        n = args ? PyTuple_GET_SIZE(args) : 0;
         newargs = PyTuple_New(n+1);
         if (newargs == NULL) {
-            Py_DECREF(args);
+            Py_XDECREF(args);
             Py_DECREF(newobj);
             return NULL;
         }
@@ -4149,7 +4169,7 @@ reduce_newobj(PyObject *obj, int proto)
             Py_INCREF(v);
             PyTuple_SET_ITEM(newargs, i+1, v);
         }
-        Py_DECREF(args);
+        Py_XDECREF(args);
     }
     else if (proto >= 4) {
         _Py_IDENTIFIER(__newobj_ex__);
@@ -4180,7 +4200,8 @@ reduce_newobj(PyObject *obj, int proto)
         return NULL;
     }
 
-    state = _PyObject_GetState(obj);
+    state = _PyObject_GetState(obj,
+                !hasargs && !PyList_Check(obj) && !PyDict_Check(obj));
     if (state == NULL) {
         Py_DECREF(newobj);
         Py_DECREF(newargs);
