Issue #10182: The re module doesn't truncate indices to 32 bits anymore.

pitrou · pitrou · commit 43fb54cd4f27 · 2012-12-02T12:52:36.000+01:00
Patch by Serhiy Storchaka.
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
@@ -1,4 +1,4 @@
-from test.support import verbose, run_unittest, gc_collect
+from test.support import verbose, run_unittest, gc_collect, bigmemtest, _2G
 import io
 import re
 from re import Scanner
@@ -854,6 +854,21 @@ def test_compile(self):
         # Test behaviour when not given a string or pattern as parameter
         self.assertRaises(TypeError, re.compile, 0)
 
+    # The huge memuse is because of re.sub() using a list and a join()
+    # to create the replacement result.
+    @bigmemtest(size=_2G, memuse=20)
+    def test_large(self, size):
+        # Issue #10182: indices were 32-bit-truncated.
+        s = 'a' * size
+        m = re.search('$', s)
+        self.assertIsNotNone(m)
+        self.assertEqual(m.start(), size)
+        self.assertEqual(m.end(), size)
+        r, n = re.subn('', '', s)
+        self.assertEqual(r, s)
+        self.assertEqual(n, size + 1)
+
+
 def run_re_tests():
     from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR
     if verbose:
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -169,6 +169,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #10182: The re module doesn't truncate indices to 32 bits anymore.
+  Patch by Serhiy Storchaka.
+
 - Issue #16573: In 2to3, treat enumerate() like a consuming call, so superfluous
   list() calls aren't added to filter(), map(), and zip() which are directly
   passed enumerate().
diff --git a/Modules/_sre.c b/Modules/_sre.c
@@ -1629,7 +1629,7 @@ static PyObject*pattern_scanner(PatternObject*, PyObject*);
 static PyObject *
 sre_codesize(PyObject* self, PyObject *unused)
 {
-    return Py_BuildValue("l", sizeof(SRE_CODE));
+    return PyLong_FromSize_t(sizeof(SRE_CODE));
 }
 
 static PyObject *
@@ -2467,7 +2467,7 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string,
         return NULL;
 
     if (subn)
-        return Py_BuildValue("Ni", item, n);
+        return Py_BuildValue("Nn", item, n);
 
     return item;
 
@@ -3423,7 +3423,7 @@ match_start(MatchObject* self, PyObject* args)
     }
 
     /* mark is -1 if group is undefined */
-    return Py_BuildValue("i", self->mark[index*2]);
+    return PyLong_FromSsize_t(self->mark[index*2]);
 }
 
 static PyObject*
@@ -3446,7 +3446,7 @@ match_end(MatchObject* self, PyObject* args)
     }
 
     /* mark is -1 if group is undefined */
-    return Py_BuildValue("i", self->mark[index*2+1]);
+    return PyLong_FromSsize_t(self->mark[index*2+1]);
 }
 
 LOCAL(PyObject*)
@@ -3596,7 +3596,7 @@ static PyObject *
 match_lastindex_get(MatchObject *self)
 {
     if (self->lastindex >= 0)
-	return Py_BuildValue("i", self->lastindex);
+        return PyLong_FromSsize_t(self->lastindex);
     Py_INCREF(Py_None);
     return Py_None;
 }

Original file line number	Diff line number	Diff line change
`@@ -1629,7 +1629,7 @@ static PyObjectpattern_scanner(PatternObject, PyObject*);`
`1629`	`1629`	`static PyObject *`
`1630`	`1630`	`sre_codesize(PyObject* self, PyObject *unused)`
`1631`	`1631`	`{`
`1632`		`- return Py_BuildValue("l", sizeof(SRE_CODE));`
	`1632`	`+ return PyLong_FromSize_t(sizeof(SRE_CODE));`
`1633`	`1633`	`}`
`1634`	`1634`
`1635`	`1635`	`static PyObject *`
`@@ -2467,7 +2467,7 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string,`
`2467`	`2467`	`return NULL;`
`2468`	`2468`
`2469`	`2469`	`if (subn)`
`2470`		`- return Py_BuildValue("Ni", item, n);`
	`2470`	`+ return Py_BuildValue("Nn", item, n);`
`2471`	`2471`
`2472`	`2472`	`return item;`
`2473`	`2473`
`@@ -3423,7 +3423,7 @@ match_start(MatchObject* self, PyObject* args)`
`3423`	`3423`	`}`
`3424`	`3424`
`3425`	`3425`	`/* mark is -1 if group is undefined */`
`3426`		`- return Py_BuildValue("i", self->mark[index*2]);`
	`3426`	`+ return PyLong_FromSsize_t(self->mark[index*2]);`
`3427`	`3427`	`}`
`3428`	`3428`
`3429`	`3429`	`static PyObject*`
`@@ -3446,7 +3446,7 @@ match_end(MatchObject* self, PyObject* args)`
`3446`	`3446`	`}`
`3447`	`3447`
`3448`	`3448`	`/* mark is -1 if group is undefined */`
`3449`		`- return Py_BuildValue("i", self->mark[index*2+1]);`
	`3449`	`+ return PyLong_FromSsize_t(self->mark[index*2+1]);`
`3450`	`3450`	`}`
`3451`	`3451`
`3452`	`3452`	`LOCAL(PyObject*)`
`@@ -3596,7 +3596,7 @@ static PyObject *`
`3596`	`3596`	`match_lastindex_get(MatchObject *self)`
`3597`	`3597`	`{`
`3598`	`3598`	`if (self->lastindex >= 0)`
`3599`		`- return Py_BuildValue("i", self->lastindex);`
	`3599`	`+ return PyLong_FromSsize_t(self->lastindex);`
`3600`	`3600`	`Py_INCREF(Py_None);`
`3601`	`3601`	`return Py_None;`
`3602`	`3602`	`}`