Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit fb1a5eb

Browse files
amauryfaamauryfa
committed
#3640: Correct a crash in cPickle on 64bit platforms, in the case of deeply nested lists or dicts.
Reviewed by Martin von Loewis.
1 parent 06974bb commit fb1a5eb

2 files changed

Lines changed: 138 additions & 62 deletions

File tree

‎Misc/find_recursionlimit.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"""
2121

2222
import sys
23+
import itertools
2324

2425
class RecursiveBlowup1:
2526
def __init__(self):
@@ -59,6 +60,24 @@ def test_getitem():
5960
def test_recurse():
6061
return test_recurse()
6162

63+
def test_cpickle(_cache={}):
64+
import io
65+
try:
66+
import _pickle
67+
except ImportError:
68+
print("cannot import _pickle, skipped!")
69+
return
70+
l = None
71+
for n in itertools.count():
72+
try:
73+
l = _cache[n]
74+
continue # Already tried and it works, let's save some time
75+
except KeyError:
76+
for i in range(100):
77+
l = [l]
78+
_pickle.Pickler(io.BytesIO(), protocol=-1).dump(l)
79+
_cache[n] = l
80+
6281
def check_limit(n, test_func_name):
6382
sys.setrecursionlimit(n)
6483
if test_func_name.startswith("test_"):
@@ -81,5 +100,6 @@ def check_limit(n, test_func_name):
81100
check_limit(limit, "test_init")
82101
check_limit(limit, "test_getattr")
83102
check_limit(limit, "test_getitem")
103+
check_limit(limit, "test_cpickle")
84104
print("Limit of %d is fine" % limit)
85105
limit = limit + 100

‎Modules/_pickle.c‎

Lines changed: 118 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1353,8 +1353,8 @@ save_tuple(PicklerObject *self, PyObject *obj)
13531353
static int
13541354
batch_list(PicklerObject *self, PyObject *iter)
13551355
{
1356-
PyObject *obj;
1357-
PyObject *slice[BATCHSIZE];
1356+
PyObject *obj = NULL;
1357+
PyObject *firstitem = NULL;
13581358
int i, n;
13591359

13601360
const char mark_op = MARK;
@@ -1389,44 +1389,69 @@ batch_list(PicklerObject *self, PyObject *iter)
13891389

13901390
/* proto > 0: write in batches of BATCHSIZE. */
13911391
do {
1392-
/* Get next group of (no more than) BATCHSIZE elements. */
1393-
for (n = 0; n < BATCHSIZE; n++) {
1392+
/* Get first item */
1393+
firstitem = PyIter_Next(iter);
1394+
if (firstitem == NULL) {
1395+
if (PyErr_Occurred())
1396+
goto error;
1397+
1398+
/* nothing more to add */
1399+
break;
1400+
}
1401+
1402+
/* Try to get a second item */
1403+
obj = PyIter_Next(iter);
1404+
if (obj == NULL) {
1405+
if (PyErr_Occurred())
1406+
goto error;
1407+
1408+
/* Only one item to write */
1409+
if (save(self, firstitem, 0) < 0)
1410+
goto error;
1411+
if (pickler_write(self, &append_op, 1) < 0)
1412+
goto error;
1413+
Py_CLEAR(firstitem);
1414+
break;
1415+
}
1416+
1417+
/* More than one item to write */
1418+
1419+
/* Pump out MARK, items, APPENDS. */
1420+
if (pickler_write(self, &mark_op, 1) < 0)
1421+
goto error;
1422+
1423+
if (save(self, firstitem, 0) < 0)
1424+
goto error;
1425+
Py_CLEAR(firstitem);
1426+
n = 1;
1427+
1428+
/* Fetch and save up to BATCHSIZE items */
1429+
while (obj) {
1430+
if (save(self, obj, 0) < 0)
1431+
goto error;
1432+
Py_CLEAR(obj);
1433+
n += 1;
1434+
1435+
if (n == BATCHSIZE)
1436+
break;
1437+
13941438
obj = PyIter_Next(iter);
13951439
if (obj == NULL) {
13961440
if (PyErr_Occurred())
13971441
goto error;
13981442
break;
13991443
}
1400-
slice[n] = obj;
14011444
}
14021445

1403-
if (n > 1) {
1404-
/* Pump out MARK, slice[0:n], APPENDS. */
1405-
if (pickler_write(self, &mark_op, 1) < 0)
1406-
goto error;
1407-
for (i = 0; i < n; i++) {
1408-
if (save(self, slice[i], 0) < 0)
1409-
goto error;
1410-
}
1411-
if (pickler_write(self, &appends_op, 1) < 0)
1412-
goto error;
1413-
}
1414-
else if (n == 1) {
1415-
if (save(self, slice[0], 0) < 0 ||
1416-
pickler_write(self, &append_op, 1) < 0)
1417-
goto error;
1418-
}
1446+
if (pickler_write(self, &appends_op, 1) < 0)
1447+
goto error;
14191448

1420-
for (i = 0; i < n; i++) {
1421-
Py_DECREF(slice[i]);
1422-
}
14231449
} while (n == BATCHSIZE);
14241450
return 0;
14251451

14261452
error:
1427-
while (--n >= 0) {
1428-
Py_DECREF(slice[n]);
1429-
}
1453+
Py_XDECREF(firstitem);
1454+
Py_XDECREF(obj);
14301455
return -1;
14311456
}
14321457

@@ -1496,8 +1521,8 @@ save_list(PicklerObject *self, PyObject *obj)
14961521
static int
14971522
batch_dict(PicklerObject *self, PyObject *iter)
14981523
{
1499-
PyObject *obj;
1500-
PyObject *slice[BATCHSIZE];
1524+
PyObject *obj = NULL;
1525+
PyObject *firstitem = NULL;
15011526
int i, n;
15021527

15031528
const char mark_op = MARK;
@@ -1534,53 +1559,84 @@ batch_dict(PicklerObject *self, PyObject *iter)
15341559

15351560
/* proto > 0: write in batches of BATCHSIZE. */
15361561
do {
1537-
/* Get next group of (no more than) BATCHSIZE elements. */
1538-
for (n = 0; n < BATCHSIZE; n++) {
1539-
obj = PyIter_Next(iter);
1540-
if (obj == NULL) {
1541-
if (PyErr_Occurred())
1542-
goto error;
1543-
break;
1544-
}
1545-
if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
1546-
PyErr_SetString(PyExc_TypeError, "dict items "
1547-
"iterator must return 2-tuples");
1562+
/* Get first item */
1563+
firstitem = PyIter_Next(iter);
1564+
if (firstitem == NULL) {
1565+
if (PyErr_Occurred())
15481566
goto error;
1549-
}
1550-
slice[n] = obj;
1567+
1568+
/* nothing more to add */
1569+
break;
1570+
}
1571+
if (!PyTuple_Check(firstitem) || PyTuple_Size(firstitem) != 2) {
1572+
PyErr_SetString(PyExc_TypeError, "dict items "
1573+
"iterator must return 2-tuples");
1574+
goto error;
15511575
}
15521576

1553-
if (n > 1) {
1554-
/* Pump out MARK, slice[0:n], SETITEMS. */
1555-
if (pickler_write(self, &mark_op, 1) < 0)
1577+
/* Try to get a second item */
1578+
obj = PyIter_Next(iter);
1579+
if (obj == NULL) {
1580+
if (PyErr_Occurred())
15561581
goto error;
1557-
for (i = 0; i < n; i++) {
1558-
obj = slice[i];
1559-
if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 ||
1560-
save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0)
1561-
goto error;
1562-
}
1563-
if (pickler_write(self, &setitems_op, 1) < 0)
1582+
1583+
/* Only one item to write */
1584+
if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1585+
goto error;
1586+
if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1587+
goto error;
1588+
if (pickler_write(self, &setitem_op, 1) < 0)
15641589
goto error;
1590+
Py_CLEAR(firstitem);
1591+
break;
15651592
}
1566-
else if (n == 1) {
1567-
obj = slice[0];
1593+
1594+
/* More than one item to write */
1595+
1596+
/* Pump out MARK, items, SETITEMS. */
1597+
if (pickler_write(self, &mark_op, 1) < 0)
1598+
goto error;
1599+
1600+
if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1601+
goto error;
1602+
if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1603+
goto error;
1604+
Py_CLEAR(firstitem);
1605+
n = 1;
1606+
1607+
/* Fetch and save up to BATCHSIZE items */
1608+
while (obj) {
1609+
if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
1610+
PyErr_SetString(PyExc_TypeError, "dict items "
1611+
"iterator must return 2-tuples");
1612+
goto error;
1613+
}
15681614
if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 ||
1569-
save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0 ||
1570-
pickler_write(self, &setitem_op, 1) < 0)
1615+
save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0)
15711616
goto error;
1572-
}
1617+
Py_CLEAR(obj);
1618+
n += 1;
1619+
1620+
if (n == BATCHSIZE)
1621+
break;
15731622

1574-
for (i = 0; i < n; i++) {
1575-
Py_DECREF(slice[i]);
1623+
obj = PyIter_Next(iter);
1624+
if (obj == NULL) {
1625+
if (PyErr_Occurred())
1626+
goto error;
1627+
break;
1628+
}
15761629
}
1630+
1631+
if (pickler_write(self, &setitems_op, 1) < 0)
1632+
goto error;
1633+
15771634
} while (n == BATCHSIZE);
15781635
return 0;
15791636

15801637
error:
1581-
while (--n >= 0) {
1582-
Py_DECREF(slice[n]);
1583-
}
1638+
Py_XDECREF(firstitem);
1639+
Py_XDECREF(obj);
15841640
return -1;
15851641
}
15861642

0 commit comments

Comments
 (0)