Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f2c1aa1

Browse files
vstinnervstinner
committed
Add ast.Constant
Issue #26146: Add a new kind of AST node: ast.Constant. It can be used by external AST optimizers, but the compiler does not emit directly such node. An optimizer can replace the following AST nodes with ast.Constant: * ast.NameConstant: None, False, True * ast.Num: int, float, complex * ast.Str: str * ast.Bytes: bytes * ast.Tuple if items are constants too: tuple * frozenset Update code to accept ast.Constant instead of ast.Num and/or ast.Str: * compiler * docstrings * ast.literal_eval() * Tools/parser/unparse.py
1 parent 0dceb91 commit f2c1aa1

14 files changed

Lines changed: 401 additions & 44 deletions

File tree

‎Include/Python-ast.h‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
202202
Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
203203
Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
204204
FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21,
205-
NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24,
206-
Subscript_kind=25, Starred_kind=26, Name_kind=27,
207-
List_kind=28, Tuple_kind=29};
205+
NameConstant_kind=22, Ellipsis_kind=23, Constant_kind=24,
206+
Attribute_kind=25, Subscript_kind=26, Starred_kind=27,
207+
Name_kind=28, List_kind=29, Tuple_kind=30};
208208
struct _expr {
209209
enum _expr_kind kind;
210210
union {
@@ -315,6 +315,10 @@ struct _expr {
315315
singleton value;
316316
} NameConstant;
317317

318+
struct {
319+
constant value;
320+
} Constant;
321+
318322
struct {
319323
expr_ty value;
320324
identifier attr;
@@ -567,6 +571,9 @@ expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena
567571
*arena);
568572
#define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
569573
expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
574+
#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3)
575+
expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena
576+
*arena);
570577
#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
571578
expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
572579
lineno, int col_offset, PyArena *arena);

‎Include/asdl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ typedef PyObject * string;
66
typedef PyObject * bytes;
77
typedef PyObject * object;
88
typedef PyObject * singleton;
9+
typedef PyObject * constant;
910

1011
/* It would be nice if the code generated by asdl_c.py was completely
1112
independent of Python, but it is a goal the requires too much work

‎Lib/ast.py‎

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ def parse(source, filename='<unknown>', mode='exec'):
3535
return compile(source, filename, mode, PyCF_ONLY_AST)
3636

3737

38+
_NUM_TYPES = (int, float, complex)
39+
3840
def literal_eval(node_or_string):
3941
"""
4042
Safely evaluate an expression node or a string containing a Python
@@ -47,7 +49,9 @@ def literal_eval(node_or_string):
4749
if isinstance(node_or_string, Expression):
4850
node_or_string = node_or_string.body
4951
def _convert(node):
50-
if isinstance(node, (Str, Bytes)):
52+
if isinstance(node, Constant):
53+
return node.value
54+
elif isinstance(node, (Str, Bytes)):
5155
return node.s
5256
elif isinstance(node, Num):
5357
return node.n
@@ -62,24 +66,21 @@ def _convert(node):
6266
in zip(node.keys, node.values))
6367
elif isinstance(node, NameConstant):
6468
return node.value
65-
elif isinstance(node, UnaryOp) and \
66-
isinstance(node.op, (UAdd, USub)) and \
67-
isinstance(node.operand, (Num, UnaryOp, BinOp)):
69+
elif isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)):
6870
operand = _convert(node.operand)
69-
if isinstance(node.op, UAdd):
70-
return + operand
71-
else:
72-
return - operand
73-
elif isinstance(node, BinOp) and \
74-
isinstance(node.op, (Add, Sub)) and \
75-
isinstance(node.right, (Num, UnaryOp, BinOp)) and \
76-
isinstance(node.left, (Num, UnaryOp, BinOp)):
71+
if isinstance(operand, _NUM_TYPES):
72+
if isinstance(node.op, UAdd):
73+
return + operand
74+
else:
75+
return - operand
76+
elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)):
7777
left = _convert(node.left)
7878
right = _convert(node.right)
79-
if isinstance(node.op, Add):
80-
return left + right
81-
else:
82-
return left - right
79+
if isinstance(left, _NUM_TYPES) and isinstance(right, _NUM_TYPES):
80+
if isinstance(node.op, Add):
81+
return left + right
82+
else:
83+
return left - right
8384
raise ValueError('malformed node or string: ' + repr(node))
8485
return _convert(node_or_string)
8586

@@ -196,12 +197,19 @@ def get_docstring(node, clean=True):
196197
"""
197198
if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
198199
raise TypeError("%r can't have docstrings" % node.__class__.__name__)
199-
if node.body and isinstance(node.body[0], Expr) and \
200-
isinstance(node.body[0].value, Str):
201-
if clean:
202-
import inspect
203-
return inspect.cleandoc(node.body[0].value.s)
204-
return node.body[0].value.s
200+
if not(node.body and isinstance(node.body[0], Expr)):
201+
return
202+
node = node.body[0].value
203+
if isinstance(node, Str):
204+
text = node.s
205+
elif isinstance(node, Constant) and isinstance(node.value, str):
206+
text = node.value
207+
else:
208+
return
209+
if clean:
210+
import inspect
211+
text = inspect.cleandoc(text)
212+
return text
205213

206214

207215
def walk(node):

‎Lib/test/test_ast.py‎

Lines changed: 119 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1+
import ast
2+
import dis
13
import os
24
import sys
35
import unittest
4-
import ast
56
import weakref
67

78
from test import support
@@ -933,6 +934,123 @@ def test_stdlib_validates(self):
933934
compile(mod, fn, "exec")
934935

935936

937+
class ConstantTests(unittest.TestCase):
938+
"""Tests on the ast.Constant node type."""
939+
940+
def compile_constant(self, value):
941+
tree = ast.parse("x = 123")
942+
943+
node = tree.body[0].value
944+
new_node = ast.Constant(value=value)
945+
ast.copy_location(new_node, node)
946+
tree.body[0].value = new_node
947+
948+
code = compile(tree, "<string>", "exec")
949+
950+
ns = {}
951+
exec(code, ns)
952+
return ns['x']
953+
954+
def test_singletons(self):
955+
for const in (None, False, True, Ellipsis, b'', frozenset()):
956+
with self.subTest(const=const):
957+
value = self.compile_constant(const)
958+
self.assertIs(value, const)
959+
960+
def test_values(self):
961+
nested_tuple = (1,)
962+
nested_frozenset = frozenset({1})
963+
for level in range(3):
964+
nested_tuple = (nested_tuple, 2)
965+
nested_frozenset = frozenset({nested_frozenset, 2})
966+
values = (123, 123.0, 123j,
967+
"unicode", b'bytes',
968+
tuple("tuple"), frozenset("frozenset"),
969+
nested_tuple, nested_frozenset)
970+
for value in values:
971+
with self.subTest(value=value):
972+
result = self.compile_constant(value)
973+
self.assertEqual(result, value)
974+
975+
def test_assign_to_constant(self):
976+
tree = ast.parse("x = 1")
977+
978+
target = tree.body[0].targets[0]
979+
new_target = ast.Constant(value=1)
980+
ast.copy_location(new_target, target)
981+
tree.body[0].targets[0] = new_target
982+
983+
with self.assertRaises(ValueError) as cm:
984+
compile(tree, "string", "exec")
985+
self.assertEqual(str(cm.exception),
986+
"expression which can't be assigned "
987+
"to in Store context")
988+
989+
def test_get_docstring(self):
990+
tree = ast.parse("'docstring'\nx = 1")
991+
self.assertEqual(ast.get_docstring(tree), 'docstring')
992+
993+
tree.body[0].value = ast.Constant(value='constant docstring')
994+
self.assertEqual(ast.get_docstring(tree), 'constant docstring')
995+
996+
def get_load_const(self, tree):
997+
# Compile to bytecode, disassemble and get parameter of LOAD_CONST
998+
# instructions
999+
co = compile(tree, '<string>', 'exec')
1000+
consts = []
1001+
for instr in dis.get_instructions(co):
1002+
if instr.opname == 'LOAD_CONST':
1003+
consts.append(instr.argval)
1004+
return consts
1005+
1006+
@support.cpython_only
1007+
def test_load_const(self):
1008+
consts = [None,
1009+
True, False,
1010+
124,
1011+
2.0,
1012+
3j,
1013+
"unicode",
1014+
b'bytes',
1015+
(1, 2, 3)]
1016+
1017+
code = '\n'.join(map(repr, consts))
1018+
code += '\n...'
1019+
1020+
code_consts = [const for const in consts
1021+
if (not isinstance(const, (str, int, float, complex))
1022+
or isinstance(const, bool))]
1023+
code_consts.append(Ellipsis)
1024+
# the compiler adds a final "LOAD_CONST None"
1025+
code_consts.append(None)
1026+
1027+
tree = ast.parse(code)
1028+
self.assertEqual(self.get_load_const(tree), code_consts)
1029+
1030+
# Replace expression nodes with constants
1031+
for expr_node, const in zip(tree.body, consts):
1032+
assert isinstance(expr_node, ast.Expr)
1033+
new_node = ast.Constant(value=const)
1034+
ast.copy_location(new_node, expr_node.value)
1035+
expr_node.value = new_node
1036+
1037+
self.assertEqual(self.get_load_const(tree), code_consts)
1038+
1039+
def test_literal_eval(self):
1040+
tree = ast.parse("1 + 2")
1041+
binop = tree.body[0].value
1042+
1043+
new_left = ast.Constant(value=10)
1044+
ast.copy_location(new_left, binop.left)
1045+
binop.left = new_left
1046+
1047+
new_right = ast.Constant(value=20)
1048+
ast.copy_location(new_right, binop.right)
1049+
binop.right = new_right
1050+
1051+
self.assertEqual(ast.literal_eval(binop), 30)
1052+
1053+
9361054
def main():
9371055
if __name__ != '__main__':
9381056
return

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ Release date: tba
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used
14+
by external AST optimizers, but the compiler does not emit directly such
15+
node.
16+
1317
- Issue #18018: Import raises ImportError instead of SystemError if a relative
1418
import is attempted without a known parent package.
1519

‎Parser/Python.asdl‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton
1+
-- ASDL's 7 builtin types are:
2+
-- identifier, int, string, bytes, object, singleton, constant
3+
--
4+
-- singleton: None, True or False
5+
-- constant can be None, whereas None means "no value" for object.
26

37
module Python
48
{
@@ -76,6 +80,7 @@ module Python
7680
| Bytes(bytes s)
7781
| NameConstant(singleton value)
7882
| Ellipsis
83+
| Constant(constant value)
7984

8085
-- the following expression can appear in assignment context
8186
| Attribute(expr value, identifier attr, expr_context ctx)

‎Parser/asdl.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@
3333
# See the EBNF at the top of the file to understand the logical connection
3434
# between the various node types.
3535

36-
builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'}
36+
builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton',
37+
'constant'}
3738

3839
class AST:
3940
def __repr__(self):

‎Parser/asdl_c.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,7 @@ def visitModule(self, mod):
834834
return (PyObject*)o;
835835
}
836836
#define ast2obj_singleton ast2obj_object
837+
#define ast2obj_constant ast2obj_object
837838
#define ast2obj_identifier ast2obj_object
838839
#define ast2obj_string ast2obj_object
839840
#define ast2obj_bytes ast2obj_object
@@ -871,6 +872,26 @@ def visitModule(self, mod):
871872
return 0;
872873
}
873874
875+
static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena)
876+
{
877+
if (obj == Py_None || obj == Py_True || obj == Py_False) {
878+
/* don't increment the reference counter, Constant uses a borrowed
879+
* reference, not a strong reference */
880+
*out = obj;
881+
return 0;
882+
}
883+
884+
if (obj) {
885+
if (PyArena_AddPyObject(arena, obj) < 0) {
886+
*out = NULL;
887+
return -1;
888+
}
889+
Py_INCREF(obj);
890+
}
891+
*out = obj;
892+
return 0;
893+
}
894+
874895
static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena)
875896
{
876897
if (!PyUnicode_CheckExact(obj) && obj != Py_None) {

0 commit comments

Comments
 (0)