Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 31668b8

Browse files
author
Kristján Valur Jónsson
committed
Issue #14288: Serialization support for builtin iterators.
1 parent 283b96b commit 31668b8

28 files changed

Lines changed: 2190 additions & 104 deletions

‎Include/iterobject.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ PyAPI_FUNC(PyObject *) PySeqIter_New(PyObject *);
1818

1919
PyAPI_FUNC(PyObject *) PyCallIter_New(PyObject *, PyObject *);
2020

21+
PyAPI_FUNC(PyObject *) _PyIter_GetBuiltin(const char *iter);
22+
2123
#ifdef __cplusplus
2224
}
2325
#endif

‎Lib/test/seq_tests.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import unittest
66
import sys
7+
import pickle
78

89
# Various iterables
910
# This is used for checking the constructor (here and in test_deque.py)
@@ -388,3 +389,9 @@ def __eq__(self, other):
388389
self.assertEqual(a.index(0, -4*sys.maxsize, 4*sys.maxsize), 2)
389390
self.assertRaises(ValueError, a.index, 0, 4*sys.maxsize,-4*sys.maxsize)
390391
self.assertRaises(ValueError, a.index, 2, 0, -10)
392+
393+
def test_pickle(self):
394+
lst = self.type2test([4, 5, 6, 7])
395+
lst2 = pickle.loads(pickle.dumps(lst))
396+
self.assertEqual(lst2, lst)
397+
self.assertNotEqual(id(lst2), id(lst))

‎Lib/test/test_array.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,20 @@ def test_pickle_for_empty_array(self):
285285
self.assertEqual(a.x, b.x)
286286
self.assertEqual(type(a), type(b))
287287

288+
def test_iterator_pickle(self):
289+
data = array.array(self.typecode, self.example)
290+
orgit = iter(data)
291+
d = pickle.dumps(orgit)
292+
it = pickle.loads(d)
293+
self.assertEqual(type(orgit), type(it))
294+
self.assertEqual(list(it), list(data))
295+
296+
if len(data):
297+
it = pickle.loads(d)
298+
next(it)
299+
d = pickle.dumps(it)
300+
self.assertEqual(list(it), list(data)[1:])
301+
288302
def test_insert(self):
289303
a = array.array(self.typecode, self.example)
290304
a.insert(0, self.example[0])

‎Lib/test/test_builtin.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import traceback
1515
from test.support import TESTFN, unlink, run_unittest, check_warnings
1616
from operator import neg
17+
import pickle
1718
try:
1819
import pty, signal
1920
except ImportError:
@@ -110,7 +111,30 @@ class TestFailingIter:
110111
def __iter__(self):
111112
raise RuntimeError
112113

114+
def filter_char(arg):
115+
return ord(arg) > ord("d")
116+
117+
def map_char(arg):
118+
return chr(ord(arg)+1)
119+
113120
class BuiltinTest(unittest.TestCase):
121+
# Helper to check picklability
122+
def check_iter_pickle(self, it, seq):
123+
itorg = it
124+
d = pickle.dumps(it)
125+
it = pickle.loads(d)
126+
self.assertEqual(type(itorg), type(it))
127+
self.assertEqual(list(it), seq)
128+
129+
#test the iterator after dropping one from it
130+
it = pickle.loads(d)
131+
try:
132+
next(it)
133+
except StopIteration:
134+
return
135+
d = pickle.dumps(it)
136+
it = pickle.loads(d)
137+
self.assertEqual(list(it), seq[1:])
114138

115139
def test_import(self):
116140
__import__('sys')
@@ -566,6 +590,11 @@ def badfunc():
566590
self.assertEqual(list(filter(lambda x: x>=3, (1, 2, 3, 4))), [3, 4])
567591
self.assertRaises(TypeError, list, filter(42, (1, 2)))
568592

593+
def test_filter_pickle(self):
594+
f1 = filter(filter_char, "abcdeabcde")
595+
f2 = filter(filter_char, "abcdeabcde")
596+
self.check_iter_pickle(f1, list(f2))
597+
569598
def test_getattr(self):
570599
self.assertTrue(getattr(sys, 'stdout') is sys.stdout)
571600
self.assertRaises(TypeError, getattr, sys, 1)
@@ -759,6 +788,11 @@ def badfunc(x):
759788
raise RuntimeError
760789
self.assertRaises(RuntimeError, list, map(badfunc, range(5)))
761790

791+
def test_map_pickle(self):
792+
m1 = map(map_char, "Is this the real life?")
793+
m2 = map(map_char, "Is this the real life?")
794+
self.check_iter_pickle(m1, list(m2))
795+
762796
def test_max(self):
763797
self.assertEqual(max('123123'), '3')
764798
self.assertEqual(max(1, 2, 3), 3)
@@ -1300,6 +1334,13 @@ def __getitem__(self, i):
13001334
return i
13011335
self.assertRaises(ValueError, list, zip(BadSeq(), BadSeq()))
13021336

1337+
def test_zip_pickle(self):
1338+
a = (1, 2, 3)
1339+
b = (4, 5, 6)
1340+
t = [(1, 4), (2, 5), (3, 6)]
1341+
z1 = zip(a, b)
1342+
self.check_iter_pickle(z1, t)
1343+
13031344
def test_format(self):
13041345
# Test the basic machinery of the format() builtin. Don't test
13051346
# the specifics of the various formatters

‎Lib/test/test_bytes.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,24 @@ def test_pickling(self):
518518
q = pickle.loads(ps)
519519
self.assertEqual(b, q)
520520

521+
def test_iterator_pickling(self):
522+
for b in b"", b"a", b"abc", b"\xffab\x80", b"\0\0\377\0\0":
523+
it = itorg = iter(self.type2test(b))
524+
data = list(self.type2test(b))
525+
d = pickle.dumps(it)
526+
it = pickle.loads(d)
527+
self.assertEqual(type(itorg), type(it))
528+
self.assertEqual(list(it), data)
529+
530+
it = pickle.loads(d)
531+
try:
532+
next(it)
533+
except StopIteration:
534+
continue
535+
d = pickle.dumps(it)
536+
it = pickle.loads(d)
537+
self.assertEqual(list(it), data[1:])
538+
521539
def test_strip(self):
522540
b = self.type2test(b'mississippi')
523541
self.assertEqual(b.strip(b'i'), b'mississipp')

‎Lib/test/test_deque.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -471,6 +471,19 @@ def test_pickle(self):
471471
## self.assertNotEqual(id(d), id(e))
472472
## self.assertEqual(id(e), id(e[-1]))
473473

474+
def test_iterator_pickle(self):
475+
data = deque(range(200))
476+
it = itorg = iter(data)
477+
d = pickle.dumps(it)
478+
it = pickle.loads(d)
479+
self.assertEqual(type(itorg), type(it))
480+
self.assertEqual(list(it), list(data))
481+
482+
it = pickle.loads(d)
483+
next(it)
484+
d = pickle.dumps(it)
485+
self.assertEqual(list(it), list(data)[1:])
486+
474487
def test_deepcopy(self):
475488
mut = [10]
476489
d = deque([mut])

‎Lib/test/test_dict.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
from test import support
33

44
import collections, random, string
5+
import collections.abc
56
import gc, weakref
7+
import pickle
68

79

810
class DictTest(unittest.TestCase):
@@ -803,6 +805,58 @@ class MyDict(dict):
803805
pass
804806
self._tracked(MyDict())
805807

808+
def test_iterator_pickling(self):
809+
data = {1:"a", 2:"b", 3:"c"}
810+
it = iter(data)
811+
d = pickle.dumps(it)
812+
it = pickle.loads(d)
813+
self.assertEqual(sorted(it), sorted(data))
814+
815+
it = pickle.loads(d)
816+
try:
817+
drop = next(it)
818+
except StopIteration:
819+
return
820+
d = pickle.dumps(it)
821+
it = pickle.loads(d)
822+
del data[drop]
823+
self.assertEqual(sorted(it), sorted(data))
824+
825+
def test_itemiterator_pickling(self):
826+
data = {1:"a", 2:"b", 3:"c"}
827+
# dictviews aren't picklable, only their iterators
828+
itorg = iter(data.items())
829+
d = pickle.dumps(itorg)
830+
it = pickle.loads(d)
831+
# note that the type of type of the unpickled iterator
832+
# is not necessarily the same as the original. It is
833+
# merely an object supporting the iterator protocol, yielding
834+
# the same objects as the original one.
835+
# self.assertEqual(type(itorg), type(it))
836+
self.assertTrue(isinstance(it, collections.abc.Iterator))
837+
self.assertEqual(dict(it), data)
838+
839+
it = pickle.loads(d)
840+
drop = next(it)
841+
d = pickle.dumps(it)
842+
it = pickle.loads(d)
843+
del data[drop[0]]
844+
self.assertEqual(dict(it), data)
845+
846+
def test_valuesiterator_pickling(self):
847+
data = {1:"a", 2:"b", 3:"c"}
848+
# data.values() isn't picklable, only its iterator
849+
it = iter(data.values())
850+
d = pickle.dumps(it)
851+
it = pickle.loads(d)
852+
self.assertEqual(sorted(list(it)), sorted(list(data.values())))
853+
854+
it = pickle.loads(d)
855+
drop = next(it)
856+
d = pickle.dumps(it)
857+
it = pickle.loads(d)
858+
values = list(it) + [drop]
859+
self.assertEqual(sorted(values), sorted(list(data.values())))
806860

807861
from test import mapping_tests
808862

‎Lib/test/test_enumerate.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import unittest
22
import sys
3+
import pickle
34

45
from test import support
56

@@ -61,7 +62,25 @@ def __init__(self, seqn):
6162
def __iter__(self):
6263
return self
6364

64-
class EnumerateTestCase(unittest.TestCase):
65+
class PickleTest:
66+
# Helper to check picklability
67+
def check_pickle(self, itorg, seq):
68+
d = pickle.dumps(itorg)
69+
it = pickle.loads(d)
70+
self.assertEqual(type(itorg), type(it))
71+
self.assertEqual(list(it), seq)
72+
73+
it = pickle.loads(d)
74+
try:
75+
next(it)
76+
except StopIteration:
77+
self.assertFalse(seq[1:])
78+
return
79+
d = pickle.dumps(it)
80+
it = pickle.loads(d)
81+
self.assertEqual(list(it), seq[1:])
82+
83+
class EnumerateTestCase(unittest.TestCase, PickleTest):
6584

6685
enum = enumerate
6786
seq, res = 'abc', [(0,'a'), (1,'b'), (2,'c')]
@@ -73,6 +92,9 @@ def test_basicfunction(self):
7392
self.assertEqual(list(self.enum(self.seq)), self.res)
7493
self.enum.__doc__
7594

95+
def test_pickle(self):
96+
self.check_pickle(self.enum(self.seq), self.res)
97+
7698
def test_getitemseqn(self):
7799
self.assertEqual(list(self.enum(G(self.seq))), self.res)
78100
e = self.enum(G(''))
@@ -126,7 +148,7 @@ class TestBig(EnumerateTestCase):
126148
seq = range(10,20000,2)
127149
res = list(zip(range(20000), seq))
128150

129-
class TestReversed(unittest.TestCase):
151+
class TestReversed(unittest.TestCase, PickleTest):
130152

131153
def test_simple(self):
132154
class A:
@@ -212,6 +234,10 @@ def __len__(self): return 2
212234
ngi = NoGetItem()
213235
self.assertRaises(TypeError, reversed, ngi)
214236

237+
def test_pickle(self):
238+
for data in 'abc', range(5), tuple(enumerate('abc')), range(1,17,5):
239+
self.check_pickle(reversed(data), list(data)[::-1])
240+
215241

216242
class EnumerateStartTestCase(EnumerateTestCase):
217243

0 commit comments

Comments
 (0)