Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 899dfba

Browse files
JelleZijlstraJelleZijlstra
authored
[3.13] gh-120108: Fix deepcopying of AST trees with .parent attributes (GH-120114) (#121000)
(cherry picked from commit 42b2c9d)
1 parent 1764a31 commit 899dfba

File tree

4 files changed

+106
-44
lines changed

4 files changed

+106
-44
lines changed

‎Lib/test/test_ast.py

Lines changed: 76 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import ast
22
import builtins
3+
import copy
34
import dis
45
import enum
56
import os
@@ -23,7 +24,7 @@
2324
from test.support.ast_helper import ASTTestMixin
2425

2526
def to_tuple(t):
26-
if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis:
27+
if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis:
2728
return t
2829
elif isinstance(t, list):
2930
return [to_tuple(e) for e in t]
@@ -971,15 +972,6 @@ def test_no_fields(self):
971972
x = ast.Sub()
972973
self.assertEqual(x._fields, ())
973974

974-
def test_pickling(self):
975-
import pickle
976-
977-
for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
978-
for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests):
979-
with self.subTest(ast=ast, protocol=protocol):
980-
ast2 = pickle.loads(pickle.dumps(ast, protocol))
981-
self.assertEqual(to_tuple(ast2), to_tuple(ast))
982-
983975
def test_invalid_sum(self):
984976
pos = dict(lineno=2, col_offset=3)
985977
m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], [])
@@ -1222,6 +1214,80 @@ def test_none_checks(self) -> None:
12221214
for node, attr, source in tests:
12231215
self.assert_none_check(node, attr, source)
12241216

1217+
1218+
class CopyTests(unittest.TestCase):
1219+
"""Test copying and pickling AST nodes."""
1220+
1221+
def test_pickling(self):
1222+
import pickle
1223+
1224+
for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
1225+
for code in exec_tests:
1226+
with self.subTest(code=code, protocol=protocol):
1227+
tree = compile(code, "?", "exec", 0x400)
1228+
ast2 = pickle.loads(pickle.dumps(tree, protocol))
1229+
self.assertEqual(to_tuple(ast2), to_tuple(tree))
1230+
1231+
def test_copy_with_parents(self):
1232+
# gh-120108
1233+
code = """
1234+
('',)
1235+
while i < n:
1236+
if ch == '':
1237+
ch = format[i]
1238+
if ch == '':
1239+
if freplace is None:
1240+
'' % getattr(object)
1241+
elif ch == '':
1242+
if zreplace is None:
1243+
if hasattr:
1244+
offset = object.utcoffset()
1245+
if offset is not None:
1246+
if offset.days < 0:
1247+
offset = -offset
1248+
h = divmod(timedelta(hours=0))
1249+
if u:
1250+
zreplace = '' % (sign,)
1251+
elif s:
1252+
zreplace = '' % (sign,)
1253+
else:
1254+
zreplace = '' % (sign,)
1255+
elif ch == '':
1256+
if Zreplace is None:
1257+
Zreplace = ''
1258+
if hasattr(object):
1259+
s = object.tzname()
1260+
if s is not None:
1261+
Zreplace = s.replace('')
1262+
newformat.append(Zreplace)
1263+
else:
1264+
push('')
1265+
else:
1266+
push(ch)
1267+
1268+
"""
1269+
tree = ast.parse(textwrap.dedent(code))
1270+
for node in ast.walk(tree):
1271+
for child in ast.iter_child_nodes(node):
1272+
child.parent = node
1273+
try:
1274+
with support.infinite_recursion(200):
1275+
tree2 = copy.deepcopy(tree)
1276+
finally:
1277+
# Singletons like ast.Load() are shared; make sure we don't
1278+
# leave them mutated after this test.
1279+
for node in ast.walk(tree):
1280+
if hasattr(node, "parent"):
1281+
del node.parent
1282+
1283+
for node in ast.walk(tree2):
1284+
for child in ast.iter_child_nodes(node):
1285+
if hasattr(child, "parent") and not isinstance(child, (
1286+
ast.expr_context, ast.boolop, ast.unaryop, ast.cmpop, ast.operator,
1287+
)):
1288+
self.assertEqual(to_tuple(child.parent), to_tuple(node))
1289+
1290+
12251291
class ASTHelpers_Test(unittest.TestCase):
12261292
maxDiff = None
12271293

‎Misc/NEWS.d/next/Library/2024-06-05-08-02-46.gh-issue-120108.4U9BL8.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix calling :func:`copy.deepcopy` on :mod:`ast` trees that have been
2+
modified to have references to parent nodes. Patch by Jelle Zijlstra.

‎Parser/asdl_c.py

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,17 +1064,22 @@ def visitModule(self, mod):
10641064
return NULL;
10651065
}
10661066
1067-
PyObject *dict = NULL, *fields = NULL, *remaining_fields = NULL,
1068-
*remaining_dict = NULL, *positional_args = NULL;
1067+
PyObject *dict = NULL, *fields = NULL, *positional_args = NULL;
10691068
if (PyObject_GetOptionalAttr(self, state->__dict__, &dict) < 0) {
10701069
return NULL;
10711070
}
10721071
PyObject *result = NULL;
10731072
if (dict) {
1074-
// Serialize the fields as positional args if possible, because if we
1075-
// serialize them as a dict, during unpickling they are set only *after*
1076-
// the object is constructed, which will now trigger a DeprecationWarning
1077-
// if the AST type has required fields.
1073+
// Unpickling (or copying) works as follows:
1074+
// - Construct the object with only positional arguments
1075+
// - Set the fields from the dict
1076+
// We have two constraints:
1077+
// - We must set all the required fields in the initial constructor call,
1078+
// or the unpickling or deepcopying of the object will trigger DeprecationWarnings.
1079+
// - We must not include child nodes in the positional args, because
1080+
// that may trigger runaway recursion during copying (gh-120108).
1081+
// To satisfy both constraints, we set all the fields to None in the
1082+
// initial list of positional args, and then set the fields from the dict.
10781083
if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {
10791084
goto cleanup;
10801085
}
@@ -1084,11 +1089,6 @@ def visitModule(self, mod):
10841089
Py_DECREF(dict);
10851090
goto cleanup;
10861091
}
1087-
remaining_dict = PyDict_Copy(dict);
1088-
Py_DECREF(dict);
1089-
if (!remaining_dict) {
1090-
goto cleanup;
1091-
}
10921092
positional_args = PyList_New(0);
10931093
if (!positional_args) {
10941094
goto cleanup;
@@ -1099,15 +1099,15 @@ def visitModule(self, mod):
10991099
goto cleanup;
11001100
}
11011101
PyObject *value;
1102-
int rc = PyDict_Pop(remaining_dict, name, &value);
1102+
int rc = PyDict_GetItemRef(dict, name, &value);
11031103
Py_DECREF(name);
11041104
if (rc < 0) {
11051105
goto cleanup;
11061106
}
11071107
if (!value) {
11081108
break;
11091109
}
1110-
rc = PyList_Append(positional_args, value);
1110+
rc = PyList_Append(positional_args, Py_None);
11111111
Py_DECREF(value);
11121112
if (rc < 0) {
11131113
goto cleanup;
@@ -1117,8 +1117,7 @@ def visitModule(self, mod):
11171117
if (!args_tuple) {
11181118
goto cleanup;
11191119
}
1120-
result = Py_BuildValue("ONO", Py_TYPE(self), args_tuple,
1121-
remaining_dict);
1120+
result = Py_BuildValue("ONN", Py_TYPE(self), args_tuple, dict);
11221121
}
11231122
else {
11241123
result = Py_BuildValue("O()N", Py_TYPE(self), dict);
@@ -1129,8 +1128,6 @@ def visitModule(self, mod):
11291128
}
11301129
cleanup:
11311130
Py_XDECREF(fields);
1132-
Py_XDECREF(remaining_fields);
1133-
Py_XDECREF(remaining_dict);
11341131
Py_XDECREF(positional_args);
11351132
return result;
11361133
}

‎Python/Python-ast.c

Lines changed: 14 additions & 17 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)