Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e2b3d83

Browse files
serhiy-storchakaserhiy-storchaka
authored
gh-109747: Improve errors for unsupported look-behind patterns (GH-109859)
Now re.error is raised instead of OverflowError or RuntimeError for too large width of look-behind pattern. The limit is increased to 2**32-1 (was 2**31-1).
1 parent ca0f3d8 commit e2b3d83

File tree

6 files changed

+46
-13
lines changed

6 files changed

+46
-13
lines changed

‎Lib/re/_compiler.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,8 @@ def _compile(code, pattern, flags):
147147
emit(0) # look ahead
148148
else:
149149
lo, hi = av[1].getwidth()
150+
if lo > MAXCODE:
151+
raise error("looks too much behind")
150152
if lo != hi:
151153
raise error("look-behind requires fixed-width pattern")
152154
emit(lo) # look behind
@@ -547,7 +549,7 @@ def _compile_info(code, pattern, flags):
547549
else:
548550
emit(MAXCODE)
549551
prefix = prefix[:MAXCODE]
550-
emit(min(hi, MAXCODE))
552+
emit(hi)
551553
# add literal prefix
552554
if prefix:
553555
emit(len(prefix)) # length

‎Lib/re/_parser.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@
6767
TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE
6868
GLOBAL_FLAGS = SRE_FLAG_DEBUG
6969

70+
# Maximal value returned by SubPattern.getwidth().
71+
# Must be larger than MAXREPEAT, MAXCODE and sys.maxsize.
72+
MAXWIDTH = 1 << 64
73+
7074
class State:
7175
# keeps track of state for parsing
7276
def __init__(self):
@@ -177,7 +181,7 @@ def getwidth(self):
177181
lo = hi = 0
178182
for op, av in self.data:
179183
if op is BRANCH:
180-
i = MAXREPEAT - 1
184+
i = MAXWIDTH
181185
j = 0
182186
for av in av[1]:
183187
l, h = av.getwidth()
@@ -196,7 +200,10 @@ def getwidth(self):
196200
elif op in _REPEATCODES:
197201
i, j = av[2].getwidth()
198202
lo = lo + i * av[0]
199-
hi = hi + j * av[1]
203+
if av[1] == MAXREPEAT and j:
204+
hi = MAXWIDTH
205+
else:
206+
hi = hi + j * av[1]
200207
elif op in _UNITCODES:
201208
lo = lo + 1
202209
hi = hi + 1
@@ -216,7 +223,7 @@ def getwidth(self):
216223
hi = hi + j
217224
elif op is SUCCESS:
218225
break
219-
self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT)
226+
self.width = min(lo, MAXWIDTH), min(hi, MAXWIDTH)
220227
return self.width
221228

222229
class Tokenizer:

‎Lib/test/test_re.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1861,6 +1861,29 @@ def test_repeat_minmax_overflow(self):
18611861
self.assertRaises(OverflowError, re.compile, r".{%d,}?" % 2**128)
18621862
self.assertRaises(OverflowError, re.compile, r".{%d,%d}" % (2**129, 2**128))
18631863

1864+
def test_look_behind_overflow(self):
1865+
string = "x" * 2_500_000
1866+
p1 = r"(?<=((.{%d}){%d}){%d})"
1867+
p2 = r"(?<!((.{%d}){%d}){%d})"
1868+
# Test that the templates are valid and look-behind with width 2**21
1869+
# (larger than sys.maxunicode) are supported.
1870+
self.assertEqual(re.search(p1 % (2**7, 2**7, 2**7), string).span(),
1871+
(2**21, 2**21))
1872+
self.assertEqual(re.search(p2 % (2**7, 2**7, 2**7), string).span(),
1873+
(0, 0))
1874+
# Test that 2**22 is accepted as a repetition number and look-behind
1875+
# width.
1876+
re.compile(p1 % (2**22, 1, 1))
1877+
re.compile(p1 % (1, 2**22, 1))
1878+
re.compile(p1 % (1, 1, 2**22))
1879+
re.compile(p2 % (2**22, 1, 1))
1880+
re.compile(p2 % (1, 2**22, 1))
1881+
re.compile(p2 % (1, 1, 2**22))
1882+
# But 2**66 is too large for look-behind width.
1883+
errmsg = "looks too much behind"
1884+
self.assertRaisesRegex(re.error, errmsg, re.compile, p1 % (2**22, 2**22, 2**22))
1885+
self.assertRaisesRegex(re.error, errmsg, re.compile, p2 % (2**22, 2**22, 2**22))
1886+
18641887
def test_backref_group_name_in_exception(self):
18651888
# Issue 17341: Poor error message when compiling invalid regex
18661889
self.checkPatternError('(?P=<foo>)',

‎Misc/NEWS.d/next/Library/2023-09-25-20-05-41.gh-issue-109747._cRJH8.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Improve errors for unsupported look-behind patterns. Now re.error is raised
2+
instead of OverflowError or RuntimeError for too large width of look-behind
3+
pattern.

‎Modules/_sre/sre.c

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2070,8 +2070,6 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups)
20702070
GET_SKIP;
20712071
GET_ARG; /* 0 for lookahead, width for lookbehind */
20722072
code--; /* Back up over arg to simplify math below */
2073-
if (arg & 0x80000000)
2074-
FAIL; /* Width too large */
20752073
/* Stop 1 before the end; we check the SUCCESS below */
20762074
if (_validate_inner(code+1, code+skip-2, groups))
20772075
FAIL;

‎Modules/_sre/sre_lib.h

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -591,8 +591,8 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
591591
/* optimization info block */
592592
/* <INFO> <1=skip> <2=flags> <3=min> ... */
593593
if (pattern[3] && (uintptr_t)(end - ptr) < pattern[3]) {
594-
TRACE(("reject (got %zd chars, need %zd)\n",
595-
end - ptr, (Py_ssize_t) pattern[3]));
594+
TRACE(("reject (got %tu chars, need %zu)\n",
595+
end - ptr, (size_t) pattern[3]));
596596
RETURN_FAILURE;
597597
}
598598
pattern += pattern[1] + 1;
@@ -1509,7 +1509,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
15091509
/* <ASSERT> <skip> <back> <pattern> */
15101510
TRACE(("|%p|%p|ASSERT %d\n", pattern,
15111511
ptr, pattern[1]));
1512-
if (ptr - (SRE_CHAR *)state->beginning < (Py_ssize_t)pattern[1])
1512+
if ((uintptr_t)(ptr - (SRE_CHAR *)state->beginning) < pattern[1])
15131513
RETURN_FAILURE;
15141514
state->ptr = ptr - pattern[1];
15151515
DO_JUMP0(JUMP_ASSERT, jump_assert, pattern+2);
@@ -1522,7 +1522,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel)
15221522
/* <ASSERT_NOT> <skip> <back> <pattern> */
15231523
TRACE(("|%p|%p|ASSERT_NOT %d\n", pattern,
15241524
ptr, pattern[1]));
1525-
if (ptr - (SRE_CHAR *)state->beginning >= (Py_ssize_t)pattern[1]) {
1525+
if ((uintptr_t)(ptr - (SRE_CHAR *)state->beginning) >= pattern[1]) {
15261526
state->ptr = ptr - pattern[1];
15271527
LASTMARK_SAVE();
15281528
if (state->repeat)
@@ -1658,9 +1658,9 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern)
16581658

16591659
flags = pattern[2];
16601660

1661-
if (pattern[3] && end - ptr < (Py_ssize_t)pattern[3]) {
1662-
TRACE(("reject (got %u chars, need %u)\n",
1663-
(unsigned int)(end - ptr), pattern[3]));
1661+
if (pattern[3] && (uintptr_t)(end - ptr) < pattern[3]) {
1662+
TRACE(("reject (got %tu chars, need %zu)\n",
1663+
end - ptr, (size_t) pattern[3]));
16641664
return 0;
16651665
}
16661666
if (pattern[3] > 1) {

0 commit comments

Comments
 (0)