From 2cdf1c7d2d6de087ab59c7a4f70b68c232f4e078 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 22 Dec 2023 14:25:25 +0000 Subject: [PATCH 1/4] GH-112215: Increase C recursion limit for non debug builds (GH-113397) --- Include/cpython/pystate.h | 4 +++- Lib/test/support/__init__.py | 14 +++++--------- Lib/test/test_functools.py | 14 ++++++++++++++ Lib/test/test_json/test_recursion.py | 6 +++--- Lib/test/test_support.py | 2 +- Lib/test/test_xml_etree.py | 2 +- .../2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst | 3 +++ 7 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 628f2e0996e469..57c19b54a6ed67 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -254,9 +254,11 @@ struct _ts { #ifndef C_RECURSION_LIMIT # ifdef __wasi__ # define C_RECURSION_LIMIT 500 +# elif defined(__s390x__) +# define C_RECURSION_LIMIT 1200 # else // This value is duplicated in Lib/test/support/__init__.py -# define C_RECURSION_LIMIT 1500 +# define C_RECURSION_LIMIT 8000 # endif #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index fd9265c93c35f3..c32d09feacec60 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2112,12 +2112,7 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=100): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" +def infinite_recursion(max_depth=20_000): if max_depth < 3: raise ValueError("max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() @@ -2356,11 +2351,12 @@ def adjust_int_max_str_digits(max_digits): finally: sys.set_int_max_str_digits(current) -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 5000 # The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 1500 +C_RECURSION_LIMIT = 8000 + +#For recursion tests, easily exceeds default recursion limit +EXCEEDS_RECURSION_LIMIT = C_RECURSION_LIMIT * 3 #Windows doesn't have os.uname() but it doesn't support s390x. skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ce2bdeb62a4da3..03fa2d7a52ca4e 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1862,6 +1862,20 @@ def orig(): ... self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') + @support.skip_on_s390x + @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + def test_lru_recursion(self): + + @self.module.lru_cache + def fib(n): + if n <= 1: + return n + return fib(n-1) + fib(n-2) + + if not support.Py_DEBUG: + with support.infinite_recursion(): + fib(2500) + @py_functools.lru_cache() def py_cached_func(x, y): diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 9919d7fbe54ef7..164ff2013eb552 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -85,10 +85,10 @@ def test_highly_nested_objects_encoding(self): for x in range(100000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(l) with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(d) def test_endless_recursion(self): @@ -99,7 +99,7 @@ def default(self, o): return [o] with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 1b78d7e84c991a..8b6abb08bb0e4c 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -633,7 +633,7 @@ def recursive_function(depth): if depth: recursive_function(depth - 1) - for max_depth in (5, 25, 250): + for max_depth in (5, 25, 250, 2500): with support.infinite_recursion(max_depth): available = support.get_recursion_available() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 53a4e9f821d6dd..a6d2d9c86afec5 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2535,7 +2535,7 @@ def __eq__(self, o): e.extend([ET.Element('bar')]) self.assertRaises(ValueError, e.remove, X('baz')) - @support.infinite_recursion(25) + @support.infinite_recursion() def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst new file mode 100644 index 00000000000000..01ca1cc7f79b8f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-15-16-26-01.gh-issue-112215.xJS6_6.rst @@ -0,0 +1,3 @@ +Increase the C recursion limit by a factor of 3 for non-debug builds, except +for webassembly and s390 platforms which are unchanged. This mitigates some +regressions in 3.12 with deep recursion mixing builtin (C) and Python code. From 0e5af41d2f9b1ec4d547b671c305f92cbf3543d0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 19:03:32 +0000 Subject: [PATCH 2/4] Reduce C recursion limit for Windows --- Include/cpython/pystate.h | 2 ++ Lib/test/test_functools.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 57c19b54a6ed67..0e3da82feb49dc 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -256,6 +256,8 @@ struct _ts { # define C_RECURSION_LIMIT 500 # elif defined(__s390x__) # define C_RECURSION_LIMIT 1200 +# elif defined(MS_WINDOWS) +# define C_RECURSION_LIMIT 7000 # else // This value is duplicated in Lib/test/support/__init__.py # define C_RECURSION_LIMIT 8000 diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 03fa2d7a52ca4e..32298e7470ca15 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1874,7 +1874,7 @@ def fib(n): if not support.Py_DEBUG: with support.infinite_recursion(): - fib(2500) + fib(2000) @py_functools.lru_cache() From 00160bc819e0d3dfa8a20522effc733aab68c4e6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 19:38:14 +0000 Subject: [PATCH 3/4] Reduce Windows C recursion limit to 6500 --- Include/cpython/pystate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0e3da82feb49dc..ece9fc20342015 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -257,7 +257,7 @@ struct _ts { # elif defined(__s390x__) # define C_RECURSION_LIMIT 1200 # elif defined(MS_WINDOWS) -# define C_RECURSION_LIMIT 7000 +# define C_RECURSION_LIMIT 6500 # else // This value is duplicated in Lib/test/support/__init__.py # define C_RECURSION_LIMIT 8000 From be8625f68e17cb07eb0665a6131ab72dcaf326c5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 15 Dec 2023 20:13:15 +0000 Subject: [PATCH 4/4] Fix #ifdefs from backport and tweak a few tests --- Include/cpython/pystate.h | 8 +++++--- Lib/test/support/__init__.py | 5 ++++- Lib/test/test_ast.py | 2 +- Lib/test/test_sys_settrace.py | 3 ++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ece9fc20342015..b7b78b26addf79 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -252,12 +252,14 @@ struct _ts { layout, optimization, and WASI runtime. Wasmtime can handle about 700 recursions, sometimes less. 500 is a more conservative limit. */ #ifndef C_RECURSION_LIMIT -# ifdef __wasi__ +# ifdef Py_DEBUG + // A debug build is likely built with low optimization level which implies + // higher stack memory usage than a release build: use a lower limit. +# define C_RECURSION_LIMIT 500 +# elif defined(__wasi__) # define C_RECURSION_LIMIT 500 # elif defined(__s390x__) # define C_RECURSION_LIMIT 1200 -# elif defined(MS_WINDOWS) -# define C_RECURSION_LIMIT 6500 # else // This value is duplicated in Lib/test/support/__init__.py # define C_RECURSION_LIMIT 8000 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c32d09feacec60..9fa2d79687f975 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2353,7 +2353,10 @@ def adjust_int_max_str_digits(max_digits): # The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 8000 +if Py_DEBUG: + C_RECURSION_LIMIT = 500 +else: + C_RECURSION_LIMIT = 8000 #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = C_RECURSION_LIMIT * 3 diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 3ba7cf7b04266f..e4df8d18ff3145 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1089,7 +1089,7 @@ def next(self): def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT crash_depth = 100_000 - success_depth = 1200 + success_depth = support.C_RECURSION_LIMIT def check_limit(prefix, repeated): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 7e16e94aa110b2..d2a4d16e96ada9 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -7,7 +7,7 @@ import gc from functools import wraps import asyncio -from test.support import import_helper +from test.support import import_helper, Py_DEBUG import contextlib import warnings @@ -2964,6 +2964,7 @@ def test_trace_unpack_long_sequence(self): counts = self.count_traces(ns["f"]) self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1}) + @unittest.skipIf(Py_DEBUG, "Debug build cannot handle deep recursion") def test_trace_lots_of_globals(self): code = """if 1: def f():