Issue #9410: Various optimizations to the pickle module, leading to

pitrou · pitrou · commit ea99c5c94985 · 2010-09-09T18:33:21.000Z
speedups up to 4x (depending on the benchmark).  Mostly ported from
Unladen Swallow; initial patch by Alexandre Vassalotti.
diff --git a/Lib/pickle.py b/Lib/pickle.py
@@ -1287,12 +1287,6 @@ def decode_long(data):
     """
     return int.from_bytes(data, byteorder='little', signed=True)
 
-# Use the faster _pickle if possible
-try:
-    from _pickle import *
-except ImportError:
-    Pickler, Unpickler = _Pickler, _Unpickler
-
 # Shorthands
 
 def dump(obj, file, protocol=None, *, fix_imports=True):
@@ -1316,6 +1310,12 @@ def loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
     return Unpickler(file, fix_imports=fix_imports,
                      encoding=encoding, errors=errors).load()
 
+# Use the faster _pickle if possible
+try:
+    from _pickle import *
+except ImportError:
+    Pickler, Unpickler = _Pickler, _Unpickler
+
 # Doctest
 def _test():
     import doctest
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
@@ -1068,6 +1068,15 @@ def test_pickle_to_2x(self):
         dumped = self.dumps(set([3]), 2)
         self.assertEqual(dumped, DATA6)
 
+    def test_large_pickles(self):
+        # Test the correctness of internal buffering routines when handling
+        # large data.
+        for proto in protocols:
+            data = (1, b'x' * (256 * 1024))
+            dumped = self.dumps(data, proto)
+            loaded = self.loads(dumped)
+            self.assertEqual(loaded, data)
+
 
 # Test classes for reduce_ex
 
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
@@ -37,6 +37,18 @@ def loads(self, buf):
         return u.load()
 
 
+class InMemoryPickleTests(AbstractPickleTests):
+
+    pickler = pickle._Pickler
+    unpickler = pickle._Unpickler
+
+    def dumps(self, arg, proto=None):
+        return pickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        return pickle.loads(buf)
+
+
 class PyPersPicklerTests(AbstractPersistentPicklerTests):
 
     pickler = pickle._Pickler
@@ -95,7 +107,8 @@ def test_main():
         tests.extend([CPicklerTests, CPersPicklerTests,
                       CDumpPickle_LoadPickle, DumpPickle_CLoadPickle,
                       PyPicklerUnpicklerObjectTests,
-                      CPicklerUnpicklerObjectTests])
+                      CPicklerUnpicklerObjectTests,
+                      InMemoryPickleTests])
     support.run_unittest(*tests)
     support.run_doctest(pickle)
 
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -20,6 +20,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #9410: Various optimizations to the pickle module, leading to
+  speedups up to 4x (depending on the benchmark).  Mostly ported from
+  Unladen Swallow; initial patch by Alexandre Vassalotti.
+
 - The pprint module now supports printing OrderedDicts in their given
   order (formerly, it would sort the keys).
 
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
