Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 154a539

Browse files
gvanrossumgvanrossum
committed
Changes for new parser module (Fred Drake)
1 parent 6e21ceb commit 154a539

3 files changed

Lines changed: 276 additions & 37 deletions

File tree

‎Lib/AST.py‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
"""Object-oriented interface to the parser module.
2+
3+
This module exports three classes which together provide an interface
4+
to the parser module. Together, the three classes represent two ways
5+
to create parsed representations of Python source and the two starting
6+
data types (source text and tuple representations). Each class
7+
provides interfaces which are identical other than the constructors.
8+
The constructors are described in detail in the documentation for each
9+
class and the remaining, shared portion of the interface is documented
10+
below. Briefly, the three classes provided are:
11+
12+
AST
13+
Defines the primary interface to the AST objects and supports creation
14+
from the tuple representation of the parse tree.
15+
16+
ExpressionAST
17+
Supports creation of expression constructs from source text.
18+
19+
SuiteAST
20+
Supports creation of statement suites from source text.
21+
22+
FileSuiteAST
23+
Convenience subclass of the `SuiteAST' class; loads source text of the
24+
suite from an external file.
25+
26+
Aside from the constructors, several methods are provided to allow
27+
access to the various interpretations of the parse tree and to check
28+
conditions of the construct represented by the parse tree.
29+
30+
ast()
31+
Returns the corresponding `parser.ASTType' object.
32+
33+
code()
34+
Returns the compiled code object.
35+
36+
filename()
37+
Returns the name of the associated source file, if known.
38+
39+
isExpression()
40+
Returns true value if parse tree represents an expression, or a false
41+
value otherwise.
42+
43+
isSuite()
44+
Returns true value if parse tree represents a suite of statements, or
45+
a false value otherwise.
46+
47+
text()
48+
Returns the source text, or None if not available.
49+
50+
tuple()
51+
Returns the tuple representing the parse tree.
52+
"""
53+
54+
__version__ = '$Revision$'
55+
__copyright__ = """Copyright (c) 1995, 1996 by Fred L. Drake, Jr.
56+
57+
This software may be used and distributed freely for any purpose provided
58+
that this notice is included unchanged on any and all copies. The author
59+
does not warrant or guarantee this software in any way.
60+
"""
61+
62+
class AST:
63+
"""Base class for Abstract Syntax Tree objects.
64+
65+
Creates an Abstract Syntax Tree based on the tuple representation
66+
of the parse tree. The parse tree can represent either an
67+
expression or a suite; which will be recognized automatically.
68+
This base class provides all of the query methods for subclass
69+
objects defined in this module.
70+
"""
71+
_p = __import__('parser') # import internally to avoid
72+
# namespace pollution at the
73+
# top level
74+
_text = None
75+
_code = None
76+
_ast = None
77+
_type = 'unknown'
78+
_tupl = None
79+
80+
def __init__(self, tuple):
81+
"""Create an `AST' instance from a tuple-tree representation.
82+
83+
tuple
84+
The tuple tree to convert.
85+
86+
The tuple-tree may represent either an expression or a suite; the
87+
type will be determined automatically.
88+
"""
89+
if type(tuple) is not type(()):
90+
raise TypeError, 'Base AST class requires tuple parameter.'
91+
92+
self._tupl = tuple
93+
self._ast = self._p.tuple2ast(tuple)
94+
self._type = (self._p.isexpr(self._ast) and 'expression') or 'suite'
95+
96+
def tuple(self):
97+
"""Returns the tuple representing the parse tree.
98+
"""
99+
if self._tupl is None:
100+
self._tupl = self._p.ast2tuple(self._ast)
101+
return self._tupl
102+
103+
def code(self):
104+
"""Returns the compiled code object.
105+
106+
The code object returned by this method may be passed to the
107+
exec statement if `AST.isSuite()' is true or to the eval()
108+
function if `AST.isExpression()' is true. All the usual rules
109+
regarding execution of code objects apply.
110+
"""
111+
if not self._code:
112+
self._code = self._p.compileast(self._ast)
113+
return self._code
114+
115+
def ast(self):
116+
"""Returns the corresponding `parser.ASTType' object.
117+
"""
118+
return self._ast
119+
120+
def filename(self):
121+
"""Returns the name of the source file if known, or None.
122+
"""
123+
return None
124+
125+
def text(self):
126+
"""Returns the source text, or None if not available.
127+
128+
If the instance is of class `AST', None is returned since no
129+
source text is available. If of class `ExpressionAST' or
130+
`SuiteAST', the source text passed to the constructor is
131+
returned.
132+
"""
133+
return self._text
134+
135+
def isSuite(self):
136+
"""Determine if `AST' instance represents a suite of statements.
137+
"""
138+
return self._type == 'suite'
139+
140+
def isExpression(self):
141+
"""Determine if `AST' instance represents an expression.
142+
"""
143+
return self._type == 'expression'
144+
145+
146+
147+
class SuiteAST(AST):
148+
"""Statement suite parse tree representation.
149+
150+
This subclass of the `AST' base class represents statement suites
151+
parsed from the source text of a Python suite. If the source text
152+
does not represent a parsable suite of statements, the appropriate
153+
exception is raised by the parser.
154+
"""
155+
_type = 'suite'
156+
157+
def __init__(self, text):
158+
"""Initialize a `SuiteAST' from source text.
159+
160+
text
161+
Source text to parse.
162+
"""
163+
if type(text) is not type(''):
164+
raise TypeError, 'SuiteAST requires source text parameter.'
165+
self._text = text
166+
self._ast = self._p.suite(text)
167+
168+
def isSuite(self):
169+
return 1
170+
171+
def isExpression(self):
172+
return 0
173+
174+
175+
class FileSuiteAST(SuiteAST):
176+
"""Representation of a python source file syntax tree.
177+
178+
This provides a convenience wrapper around the `SuiteAST' class to
179+
load the source text from an external file.
180+
"""
181+
def __init__(self, fileName):
182+
"""Initialize a `SuiteAST' from a source file.
183+
184+
fileName
185+
Name of the external source file.
186+
"""
187+
self._fileName = fileName
188+
SuiteAST.__init__(self, open(fileName).read())
189+
190+
def filename(self):
191+
return self._fileName
192+
193+
194+
195+
class ExpressionAST(AST):
196+
"""Expression parse tree representation.
197+
198+
This subclass of the `AST' base class represents expression
199+
constructs parsed from the source text of a Python expression. If
200+
the source text does not represent a parsable expression, the
201+
appropriate exception is raised by the Python parser.
202+
"""
203+
_type = 'expression'
204+
205+
def __init__(self, text):
206+
"""Initialize an expression AST from source text.
207+
208+
text
209+
Source text to parse.
210+
"""
211+
if type(text) is not type(''):
212+
raise TypeError, 'ExpressionAST requires source text parameter.'
213+
self._text = text
214+
self._ast = self._p.expr(text)
215+
216+
def isSuite(self):
217+
return 0
218+
219+
def isExpression(self):
220+
return 1
221+
222+
223+
#
224+
# end of file

‎Lib/symbol.py‎

Lines changed: 37 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -21,40 +21,45 @@
2121
return_stmt = 274
2222
raise_stmt = 275
2323
import_stmt = 276
24-
global_stmt = 277
25-
access_stmt = 278
26-
accesstype = 279
27-
exec_stmt = 280
28-
compound_stmt = 281
29-
if_stmt = 282
30-
while_stmt = 283
31-
for_stmt = 284
32-
try_stmt = 285
33-
except_clause = 286
34-
suite = 287
35-
test = 288
36-
and_test = 289
37-
not_test = 290
38-
comparison = 291
39-
comp_op = 292
40-
expr = 293
41-
xor_expr = 294
42-
and_expr = 295
43-
shift_expr = 296
44-
arith_expr = 297
45-
term = 298
46-
factor = 299
47-
atom = 300
48-
lambdef = 301
49-
trailer = 302
50-
subscript = 303
51-
exprlist = 304
52-
testlist = 305
53-
dictmaker = 306
54-
classdef = 307
24+
dotted_name = 277
25+
global_stmt = 278
26+
access_stmt = 279
27+
accesstype = 280
28+
exec_stmt = 281
29+
compound_stmt = 282
30+
if_stmt = 283
31+
while_stmt = 284
32+
for_stmt = 285
33+
try_stmt = 286
34+
except_clause = 287
35+
suite = 288
36+
test = 289
37+
and_test = 290
38+
not_test = 291
39+
comparison = 292
40+
comp_op = 293
41+
expr = 294
42+
xor_expr = 295
43+
and_expr = 296
44+
shift_expr = 297
45+
arith_expr = 298
46+
term = 299
47+
factor = 300
48+
power = 301
49+
atom = 302
50+
lambdef = 303
51+
trailer = 304
52+
subscript = 305
53+
exprlist = 306
54+
testlist = 307
55+
dictmaker = 308
56+
classdef = 309
57+
arglist = 310
58+
argument = 311
5559

5660
names = dir()
5761
sym_name = {}
5862
for name in names:
59-
number = eval(name)
63+
number = eval(name)
64+
if type(number) is type(0):
6065
sym_name[number] = name

‎Lib/token.py‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,25 @@
3636
CIRCUMFLEX = 33
3737
LEFTSHIFT = 34
3838
RIGHTSHIFT = 35
39-
OP = 36
40-
ERRORTOKEN = 37
39+
DOUBLESTAR = 36
40+
OP = 37
41+
ERRORTOKEN = 38
4142

4243
names = dir()
4344
tok_name = {}
4445
for name in names:
45-
number = eval(name)
46+
number = eval(name)
47+
if type(number) is type(0):
4648
tok_name[number] = name
4749

48-
N_TOKENS = 38 # Number of tokens including ERRORTOKEN
49-
50+
N_TOKENS = 39 # Number of tokens including ERRORTOKEN
5051
NT_OFFSET = 256 # Start of non-terminal symbols
52+
53+
def ISTERMINAL(x):
54+
return x < NT_OFFSET
55+
56+
def ISNONTERMINAL(x):
57+
return x >= NT_OFFSET
58+
59+
def ISEOF(x):
60+
return x == ENDMARKER

0 commit comments

Comments
 (0)