Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 3ce5d92

Browse files
nnorwitznnorwitz
committed
Closes release blocker #3627.
Merged revisions 65335 via svnmerge from svn+ssh://[email protected]/python/trunk TESTED=./python -E -tt ./Lib/test/regrtest.py -uall (both debug and opt) ........ r65335 | neal.norwitz | 2008-07-31 10:17:14 -0700 (Thu, 31 Jul 2008) | 1 line Security patches from Apple: prevent int overflow when allocating memory ........
1 parent 06db799 commit 3ce5d92

11 files changed

Lines changed: 201 additions & 26 deletions

File tree

‎Lib/test/seq_tests.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -304,11 +304,13 @@ def test_repeat(self):
304304
self.assertEqual(id(s), id(s*1))
305305

306306
def test_bigrepeat(self):
307-
x = self.type2test([0])
308-
x *= 2**16
309-
self.assertRaises(MemoryError, x.__mul__, 2**16)
310-
if hasattr(x, '__imul__'):
311-
self.assertRaises(MemoryError, x.__imul__, 2**16)
307+
import sys
308+
if sys.maxsize <= 2147483647:
309+
x = self.type2test([0])
310+
x *= 2**16
311+
self.assertRaises(MemoryError, x.__mul__, 2**16)
312+
if hasattr(x, '__imul__'):
313+
self.assertRaises(MemoryError, x.__imul__, 2**16)
312314

313315
def test_subscript(self):
314316
a = self.type2test([10, 11])

‎Lib/test/support.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ def import_module(name, deprecated=False):
6868
use_resources = None # Flag set to [] by regrtest.py
6969
max_memuse = 0 # Disable bigmem tests (they will still be run with
7070
# small sizes, to make sure they work.)
71+
real_max_memuse = 0
7172

7273
# _original_stdout is meant to hold stdout at the time regrtest began.
7374
# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.
@@ -599,12 +600,14 @@ def inner(*args, **kwds):
599600
_1M = 1024*1024
600601
_1G = 1024 * _1M
601602
_2G = 2 * _1G
603+
_4G = 4 * _1G
602604

603605
MAX_Py_ssize_t = sys.maxsize
604606

605607
def set_memlimit(limit):
606608
import re
607609
global max_memuse
610+
global real_max_memuse
608611
sizes = {
609612
'k': 1024,
610613
'm': _1M,
@@ -616,6 +619,7 @@ def set_memlimit(limit):
616619
if m is None:
617620
raise ValueError('Invalid memory limit %r' % (limit,))
618621
memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
622+
real_max_memuse = memlimit
619623
if memlimit > MAX_Py_ssize_t:
620624
memlimit = MAX_Py_ssize_t
621625
if memlimit < _2G - 1:
@@ -661,6 +665,27 @@ def wrapper(self):
661665
return wrapper
662666
return decorator
663667

668+
def precisionbigmemtest(size, memuse, overhead=5*_1M):
669+
def decorator(f):
670+
def wrapper(self):
671+
if not real_max_memuse:
672+
maxsize = 5147
673+
else:
674+
maxsize = size
675+
676+
if real_max_memuse and real_max_memuse < maxsize * memuse:
677+
if verbose:
678+
sys.stderr.write("Skipping %s because of memory "
679+
"constraint\n" % (f.__name__,))
680+
return
681+
682+
return f(self, maxsize)
683+
wrapper.size = size
684+
wrapper.memuse = memuse
685+
wrapper.overhead = overhead
686+
return wrapper
687+
return decorator
688+
664689
def bigaddrspacetest(f):
665690
"""Decorator for tests that fill the address space."""
666691
def wrapper(self):

‎Lib/test/test_bigmem.py‎

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from test import support
2-
from test.support import bigmemtest, _1G, _2G
2+
from test.support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest
33

44
import unittest
55
import operator
@@ -53,6 +53,22 @@ def test_center(self, size):
5353
self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
5454
self.assertEquals(s.strip(), SUBSTR.strip())
5555

56+
@precisionbigmemtest(size=_2G - 1, memuse=1)
57+
def test_center_unicode(self, size):
58+
SUBSTR = ' abc def ghi'
59+
try:
60+
s = SUBSTR.center(size)
61+
except OverflowError:
62+
pass # acceptable on 32-bit
63+
else:
64+
self.assertEquals(len(s), size)
65+
lpadsize = rpadsize = (len(s) - len(SUBSTR)) // 2
66+
if len(s) % 2:
67+
lpadsize += 1
68+
self.assertEquals(s[lpadsize:-rpadsize], SUBSTR)
69+
self.assertEquals(s.strip(), SUBSTR.strip())
70+
del s
71+
5672
@bigmemtest(minsize=_2G, memuse=2)
5773
def test_count(self, size):
5874
SUBSTR = ' abc def ghi'
@@ -69,10 +85,51 @@ def test_decode(self, size):
6985
s = b'.' * size
7086
self.assertEquals(len(s.decode('utf-8')), size)
7187

88+
def basic_encode_test(self, size, enc, c='.', expectedsize=None):
89+
if expectedsize is None:
90+
expectedsize = size
91+
92+
s = c * size
93+
self.assertEquals(len(s.encode(enc)), expectedsize)
94+
7295
@bigmemtest(minsize=_2G + 2, memuse=3)
7396
def test_encode(self, size):
74-
s = '.' * size
75-
self.assertEquals(len(s.encode('utf-8')), size)
97+
return self.basic_encode_test(size, 'utf-8')
98+
99+
@precisionbigmemtest(size=_4G / 6 + 2, memuse=2)
100+
def test_encode_raw_unicode_escape(self, size):
101+
try:
102+
return self.basic_encode_test(size, 'raw_unicode_escape')
103+
except MemoryError:
104+
pass # acceptable on 32-bit
105+
106+
@precisionbigmemtest(size=_4G / 5 + 70, memuse=3)
107+
def test_encode_utf7(self, size):
108+
try:
109+
return self.basic_encode_test(size, 'utf7')
110+
except MemoryError:
111+
pass # acceptable on 32-bit
112+
113+
@precisionbigmemtest(size=_4G / 4 + 5, memuse=6)
114+
def test_encode_utf32(self, size):
115+
try:
116+
return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4)
117+
except MemoryError:
118+
pass # acceptable on 32-bit
119+
120+
@precisionbigmemtest(size=_2G-1, memuse=2)
121+
def test_decodeascii(self, size):
122+
return self.basic_encode_test(size, 'ascii', c='A')
123+
124+
@precisionbigmemtest(size=_4G / 5, memuse=6+2)
125+
def test_unicode_repr_oflw(self, size):
126+
try:
127+
s = "\uAAAA"*size
128+
r = repr(s)
129+
except MemoryError:
130+
pass # acceptable on 32-bit
131+
else:
132+
self.failUnless(s == eval(r))
76133

77134
@bigmemtest(minsize=_2G, memuse=2)
78135
def test_endswith(self, size):
@@ -458,6 +515,11 @@ def test_repr_large(self, size):
458515
self.assertEquals(s.count('\\'), size)
459516
self.assertEquals(s.count('0'), size * 2)
460517

518+
@bigmemtest(minsize=2**32 / 5, memuse=6+2)
519+
def test_unicode_repr(self, size):
520+
s = "\uAAAA" * size
521+
self.failUnless(len(repr(s)) > size)
522+
461523
# This test is meaningful even with size < 2G, as long as the
462524
# doubled string is > 2G (but it tests more if both are > 2G :)
463525
@bigmemtest(minsize=_1G + 2, memuse=3)
@@ -641,6 +703,35 @@ def test_repeat_small(self, size):
641703
def test_repeat_large(self, size):
642704
return self.basic_test_repeat(size)
643705

706+
@bigmemtest(minsize=_1G - 1, memuse=12)
707+
def test_repeat_large_2(self, size):
708+
return self.basic_test_repeat(size)
709+
710+
@precisionbigmemtest(size=_1G - 1, memuse=9)
711+
def test_from_2G_generator(self, size):
712+
try:
713+
t = tuple(range(size))
714+
except MemoryError:
715+
pass # acceptable on 32-bit
716+
else:
717+
count = 0
718+
for item in t:
719+
self.assertEquals(item, count)
720+
count += 1
721+
self.assertEquals(count, size)
722+
723+
@precisionbigmemtest(size=_1G - 25, memuse=9)
724+
def test_from_almost_2G_generator(self, size):
725+
try:
726+
t = tuple(range(size))
727+
count = 0
728+
for item in t:
729+
self.assertEquals(item, count)
730+
count += 1
731+
self.assertEquals(count, size)
732+
except MemoryError:
733+
pass # acceptable, expected on 32-bit
734+
644735
# Like test_concat, split in two.
645736
def basic_test_repr(self, size):
646737
t = (0,) * size

‎Misc/NEWS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ What's New in Python 3.0 release candidate 1
1212
Core and Builtins
1313
-----------------
1414

15+
- Apply security patches from Apple.
16+
1517
- Fix crashes on memory allocation failure found with failmalloc.
1618

1719
- Fix memory leaks found with valgrind and update suppressions file.

‎Modules/gcmodule.c‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1318,7 +1318,10 @@ PyObject *
13181318
_PyObject_GC_Malloc(size_t basicsize)
13191319
{
13201320
PyObject *op;
1321-
PyGC_Head *g = (PyGC_Head *)PyObject_MALLOC(
1321+
PyGC_Head *g;
1322+
if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
1323+
return PyErr_NoMemory();
1324+
g = (PyGC_Head *)PyObject_MALLOC(
13221325
sizeof(PyGC_Head) + basicsize);
13231326
if (g == NULL)
13241327
return PyErr_NoMemory();
@@ -1361,6 +1364,8 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems)
13611364
{
13621365
const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems);
13631366
PyGC_Head *g = AS_GC(op);
1367+
if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head))
1368+
return (PyVarObject *)PyErr_NoMemory();
13641369
g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize);
13651370
if (g == NULL)
13661371
return (PyVarObject *)PyErr_NoMemory();

‎Modules/mmapmodule.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ mmap_read_method(mmap_object *self,
245245
return(NULL);
246246

247247
/* silently 'adjust' out-of-range requests */
248-
if ((self->pos + num_bytes) > self->size) {
248+
if (num_bytes > self->size - self->pos) {
249249
num_bytes -= (self->pos+num_bytes) - self->size;
250250
}
251251
result = PyByteArray_FromStringAndSize(self->data+self->pos, num_bytes);

‎Objects/bytearrayobject.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,11 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size)
121121
return NULL;
122122
}
123123

124+
/* Prevent buffer overflow when setting alloc to size+1. */
125+
if (size == PY_SSIZE_T_MAX) {
126+
return PyErr_NoMemory();
127+
}
128+
124129
new = PyObject_New(PyByteArrayObject, &PyByteArray_Type);
125130
if (new == NULL)
126131
return NULL;

‎Objects/bytesobject.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,12 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size)
8383
return (PyObject *)op;
8484
}
8585

86+
if (size > PY_SSIZE_T_MAX - sizeof(PyBytesObject)) {
87+
PyErr_SetString(PyExc_OverflowError,
88+
"byte string is too large");
89+
return NULL;
90+
}
91+
8692
/* Inline PyObject_NewVar */
8793
op = (PyBytesObject *)PyObject_MALLOC(sizeof(PyBytesObject) + size);
8894
if (op == NULL)
@@ -111,7 +117,7 @@ PyBytes_FromString(const char *str)
111117

112118
assert(str != NULL);
113119
size = strlen(str);
114-
if (size > PY_SSIZE_T_MAX) {
120+
if (size > PY_SSIZE_T_MAX - sizeof(PyBytesObject)) {
115121
PyErr_SetString(PyExc_OverflowError,
116122
"byte string is too long");
117123
return NULL;

‎Objects/longobject.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,8 @@ _PyLong_New(Py_ssize_t size)
139139
PyErr_NoMemory();
140140
return NULL;
141141
}
142+
/* XXX(nnorwitz): This can overflow --
143+
PyObject_NEW_VAR / _PyObject_VAR_SIZE need to detect overflow */
142144
return (PyLongObject*)PyObject_INIT_VAR(result, &PyLong_Type, size);
143145
}
144146

‎Objects/tupleobject.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,12 @@ PyTuple_New(register Py_ssize_t size)
6060
Py_ssize_t nbytes = size * sizeof(PyObject *);
6161
/* Check for overflow */
6262
if (nbytes / sizeof(PyObject *) != (size_t)size ||
63-
(nbytes += sizeof(PyTupleObject) - sizeof(PyObject *))
64-
<= 0)
63+
(nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *)))
6564
{
6665
return PyErr_NoMemory();
6766
}
67+
nbytes += sizeof(PyTupleObject) - sizeof(PyObject *);
68+
6869
op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size);
6970
if (op == NULL)
7071
return NULL;

0 commit comments

Comments
 (0)