Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit accb62b

Browse files
jeremyhyltonjeremyhylton
committed
SF patch [ 597919 ] compiler package and SET_LINENO
A variety of changes from Michael Hudson to get the compiler working with 2.3. The primary change is the handling of SET_LINENO: # The set_lineno() function and the explicit emit() calls for # SET_LINENO below are only used to generate the line number table. # As of Python 2.3, the interpreter does not have a SET_LINENO # instruction. pyassem treats SET_LINENO opcodes as a special case. A few other small changes: - Remove unused code from pycodegen and pyassem. - Fix error handling in parsermodule. When PyParser_SimplerParseString() fails, it sets an exception with detailed info. The parsermodule was clobbering that exception and replacing it was a generic "could not parse string" exception. Keep the original exception.
1 parent 436eadd commit accb62b

6 files changed

Lines changed: 79 additions & 50 deletions

File tree

‎Lib/compiler/pyassem.py‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,8 @@
66
import types
77

88
from compiler import misc
9-
from compiler.consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, \
10-
CO_VARKEYWORDS
11-
12-
def xxx_sort(l):
13-
l = l[:]
14-
def sorter(a, b):
15-
return cmp(a.bid, b.bid)
16-
l.sort(sorter)
17-
return l
9+
from compiler.consts \
10+
import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
1811

1912
class FlowGraph:
2013
def __init__(self):
@@ -77,7 +70,7 @@ def _disable_debug(self):
7770
def emit(self, *inst):
7871
if self._debug:
7972
print "\t", inst
80-
if inst[0] == 'RETURN_VALUE':
73+
if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']:
8174
self.current.addOutEdge(self.exit)
8275
if len(inst) == 2 and isinstance(inst[1], Block):
8376
self.current.addOutEdge(inst[1])
@@ -266,7 +259,7 @@ def addNext(self, block):
266259
self.next.append(block)
267260
assert len(self.next) == 1, map(str, self.next)
268261

269-
_uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
262+
_uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE',
270263
'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
271264

272265
def pruneNext(self):
@@ -443,7 +436,7 @@ def flattenGraph(self):
443436
insts.append(inst)
444437
if len(inst) == 1:
445438
pc = pc + 1
446-
else:
439+
elif inst[0] != "SET_LINENO":
447440
# arg takes 2 bytes
448441
pc = pc + 3
449442
end[b] = pc
@@ -452,7 +445,7 @@ def flattenGraph(self):
452445
inst = insts[i]
453446
if len(inst) == 1:
454447
pc = pc + 1
455-
else:
448+
elif inst[0] != "SET_LINENO":
456449
pc = pc + 3
457450
opname = inst[0]
458451
if self.hasjrel.has_elt(opname):
@@ -580,6 +573,7 @@ def makeByteCode(self):
580573
oparg = t[1]
581574
if opname == "SET_LINENO":
582575
lnotab.nextLine(oparg)
576+
continue
583577
hi, lo = twobyte(oparg)
584578
try:
585579
lnotab.addCode(self.opnum[opname], lo, hi)
@@ -697,7 +691,7 @@ def nextLine(self, lineno):
697691
# after the loading of "b". This works with the C Python
698692
# compiler because it only generates a SET_LINENO instruction
699693
# for the assignment.
700-
if line > 0:
694+
if line >= 0:
701695
push = self.lnotab.append
702696
while addr > 255:
703697
push(255); push(0)
@@ -768,6 +762,7 @@ def findDepth(self, insts, debug=0):
768762
# PRINT_EXPR?
769763
'PRINT_ITEM': -1,
770764
'RETURN_VALUE': -1,
765+
'YIELD_VALUE': -1,
771766
'EXEC_STMT': -3,
772767
'BUILD_CLASS': -2,
773768
'STORE_NAME': -1,

‎Lib/compiler/pycodegen.py‎

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
CO_NESTED, CO_GENERATOR, CO_GENERATOR_ALLOWED, CO_FUTURE_DIVISION
1414
from compiler.pyassem import TupleArg
1515

16+
# XXX The version-specific code can go, since this code only works with 2.x.
1617
# Do we have Python 1.x or Python 2.x?
1718
try:
1819
VERSION = sys.version_info[0]
@@ -32,22 +33,14 @@
3233
TRY_FINALLY = 3
3334
END_FINALLY = 4
3435

35-
# XXX this doesn't seem to be used
36-
class BlockStack(misc.Stack):
37-
__super_init = misc.Stack.__init__
38-
39-
def __init__(self):
40-
self.__super_init(self)
41-
self.loop = None
42-
4336
def compileFile(filename, display=0):
44-
f = open(filename)
37+
f = open(filename, 'U')
4538
buf = f.read()
4639
f.close()
4740
mod = Module(buf, filename)
4841
try:
4942
mod.compile(display)
50-
except SyntaxError, err:
43+
except SyntaxError:
5144
raise
5245
else:
5346
f = open(filename + "c", "wb")
@@ -134,7 +127,7 @@ def getPycHeader(self):
134127
# to indicate the type of the value. simplest way to get the
135128
# same effect is to call marshal and then skip the code.
136129
mtime = os.path.getmtime(self.filename)
137-
mtime = struct.pack('i', mtime)
130+
mtime = struct.pack('<i', mtime)
138131
return self.MAGIC + mtime
139132

140133
class LocalNameFinder:
@@ -310,9 +303,17 @@ def _implicitNameOp(self, prefix, name):
310303
else:
311304
self.emit(prefix + '_NAME', name)
312305

313-
def set_lineno(self, node, force=0):
314-
"""Emit SET_LINENO if node has lineno attribute and it is
315-
different than the last lineno emitted.
306+
# The set_lineno() function and the explicit emit() calls for
307+
# SET_LINENO below are only used to generate the line number table.
308+
# As of Python 2.3, the interpreter does not have a SET_LINENO
309+
# instruction. pyassem treats SET_LINENO opcodes as a special case.
310+
311+
def set_lineno(self, node, force=False):
312+
"""Emit SET_LINENO if necessary.
313+
314+
The instruction is considered necessary if the node has a
315+
lineno attribute and it is different than the last lineno
316+
emitted.
316317
317318
Returns true if SET_LINENO was emitted.
318319
@@ -326,8 +327,8 @@ def set_lineno(self, node, force=0):
326327
or force):
327328
self.emit('SET_LINENO', lineno)
328329
self.last_lineno = lineno
329-
return 1
330-
return 0
330+
return True
331+
return False
331332

332333
# The first few visitor methods handle nodes that generator new
333334
# code objects. They use class attributes to determine what
@@ -387,9 +388,6 @@ def _visitFuncOrLambda(self, node, isLambda=0):
387388
def visitClass(self, node):
388389
gen = self.ClassGen(node, self.scopes,
389390
self.get_module())
390-
if node.doc:
391-
self.emit('LOAD_CONST', node.doc)
392-
self.storeName('__doc__')
393391
walk(node.code, gen)
394392
gen.finish()
395393
self.set_lineno(node)
@@ -447,7 +445,7 @@ def visitWhile(self, node):
447445
self.nextBlock(loop)
448446
self.setups.push((LOOP, loop))
449447

450-
self.set_lineno(node, force=1)
448+
self.set_lineno(node, force=True)
451449
self.visit(node.test)
452450
self.emit('JUMP_IF_FALSE', else_ or after)
453451

@@ -617,7 +615,7 @@ def visitListCompFor(self, node):
617615
return start, anchor
618616

619617
def visitListCompIf(self, node, branch):
620-
self.set_lineno(node, force=1)
618+
self.set_lineno(node, force=True)
621619
self.visit(node.test)
622620
self.emit('JUMP_IF_FALSE', branch)
623621
self.newBlock()
@@ -975,7 +973,7 @@ def visitReturn(self, node):
975973
def visitYield(self, node):
976974
self.set_lineno(node)
977975
self.visit(node.value)
978-
self.emit('YIELD_STMT')
976+
self.emit('YIELD_VALUE')
979977

980978
# slice and subscript stuff
981979

@@ -1266,9 +1264,8 @@ def __init__(self, func, scopes, isLambda, class_name, mod):
12661264
self.__super_init(func, scopes, isLambda, class_name, mod)
12671265
self.graph.setFreeVars(self.scope.get_free_vars())
12681266
self.graph.setCellVars(self.scope.get_cell_vars())
1269-
if self.graph.checkFlag(CO_GENERATOR_ALLOWED):
1270-
if self.scope.generator is not None:
1271-
self.graph.setFlag(CO_GENERATOR)
1267+
if self.scope.generator is not None:
1268+
self.graph.setFlag(CO_GENERATOR)
12721269

12731270
class AbstractClassCode:
12741271

@@ -1304,6 +1301,12 @@ def __init__(self, klass, scopes, module):
13041301
self.__super_init(klass, scopes, module)
13051302
self.graph.setFreeVars(self.scope.get_free_vars())
13061303
self.graph.setCellVars(self.scope.get_cell_vars())
1304+
self.set_lineno(klass)
1305+
self.emit("LOAD_GLOBAL", "__name__")
1306+
self.storeName("__module__")
1307+
if klass.doc:
1308+
self.emit("LOAD_CONST", klass.doc)
1309+
self.storeName('__doc__')
13071310

13081311
def generateArgList(arglist):
13091312
"""Generate an arg list marking TupleArgs"""
@@ -1379,7 +1382,5 @@ def wrap_aug(node):
13791382
return wrapper[node.__class__](node)
13801383

13811384
if __name__ == "__main__":
1382-
import sys
1383-
13841385
for file in sys.argv[1:]:
13851386
compileFile(file)

‎Lib/compiler/symbols.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,9 @@ def visitClass(self, node, parent):
249249
scope = ClassScope(node.name, self.module)
250250
if parent.nested or isinstance(parent, FunctionScope):
251251
scope.nested = 1
252+
if node.doc is not None:
253+
scope.add_def('__doc__')
254+
scope.add_def('__module__')
252255
self.scopes[node] = scope
253256
prev = self.klass
254257
self.klass = node.name

‎Lib/compiler/transformer.py‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,11 @@
3737

3838
def parseFile(path):
3939
f = open(path)
40-
src = f.read()
40+
# XXX The parser API tolerates files without a trailing newline,
41+
# but not strings without a trailing newline. Always add an extra
42+
# newline to the file contents, since we're going through the string
43+
# version of the API.
44+
src = f.read() + "\n"
4145
f.close()
4246
return parse(src)
4347

@@ -100,6 +104,7 @@ def __init__(self):
100104
token.STRING: self.atom_string,
101105
token.NAME: self.atom_name,
102106
}
107+
self.encoding = None
103108

104109
def transform(self, tree):
105110
"""Transform an AST into a modified parse tree."""
@@ -110,6 +115,7 @@ def transform(self, tree):
110115
def parsesuite(self, text):
111116
"""Return a modified parse tree for the given suite text."""
112117
# Hack for handling non-native line endings on non-DOS like OSs.
118+
# this can go now we have universal newlines?
113119
text = text.replace('\x0d', '')
114120
return self.transform(parser.suite(text))
115121

@@ -131,6 +137,12 @@ def parsefile(self, file):
131137
def compile_node(self, node):
132138
### emit a line-number node?
133139
n = node[0]
140+
141+
if n == symbol.encoding_decl:
142+
self.encoding = node[2]
143+
node = node[1]
144+
n = node[0]
145+
134146
if n == symbol.single_input:
135147
return self.single_input(node[1:])
136148
if n == symbol.file_input:
@@ -519,6 +531,7 @@ def testlist(self, nodelist):
519531
return self.com_binary(Tuple, nodelist)
520532

521533
testlist_safe = testlist # XXX
534+
testlist1 = testlist
522535
exprlist = testlist
523536

524537
def test(self, nodelist):
@@ -637,11 +650,14 @@ def term(self, nodelist):
637650
def factor(self, nodelist):
638651
elt = nodelist[0]
639652
t = elt[0]
653+
print "source", nodelist[-1]
640654
node = self.com_node(nodelist[-1])
655+
# need to handle (unary op)constant here...
641656
if t == token.PLUS:
642657
node = UnaryAdd(node)
643658
node.lineno = elt[2]
644659
elif t == token.MINUS:
660+
print node
645661
node = UnarySub(node)
646662
node.lineno = elt[2]
647663
elif t == token.TILDE:
@@ -699,11 +715,21 @@ def atom_number(self, nodelist):
699715
n.lineno = nodelist[0][2]
700716
return n
701717

718+
def decode_literal(self, lit):
719+
if self.encoding:
720+
# this is particularly fragile & a bit of a
721+
# hack... changes in compile.c:parsestr and
722+
# tokenizer.c must be reflected here.
723+
if self.encoding not in ['utf-8', 'iso-8859-1']:
724+
lit = unicode(lit, 'utf-8').encode(self.encoding)
725+
return eval("# coding: %s\n%s" % (self.encoding, lit))
726+
else:
727+
return eval(lit)
728+
702729
def atom_string(self, nodelist):
703-
### need to verify this matches compile.c
704730
k = ''
705731
for node in nodelist:
706-
k = k + eval(node[1])
732+
k += self.decode_literal(node[1])
707733
n = Const(k)
708734
n.lineno = nodelist[0][2]
709735
return n

‎Modules/parsermodule.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ node2tuple(node *n, /* node to convert */
8989
PyObject *v;
9090
PyObject *w;
9191

92-
v = mkseq(1 + NCH(n));
92+
v = mkseq(1 + NCH(n) + (TYPE(n) == encoding_decl));
9393
if (v == NULL)
9494
return (v);
9595
w = PyInt_FromLong(TYPE(n));
@@ -106,6 +106,9 @@ node2tuple(node *n, /* node to convert */
106106
}
107107
(void) addelem(v, i+1, w);
108108
}
109+
110+
if (TYPE(n) == encoding_decl)
111+
(void) addelem(v, i+1, PyString_FromString(STR(n)));
109112
return (v);
110113
}
111114
else if (ISTERMINAL(TYPE(n))) {
@@ -478,7 +481,7 @@ err_string(char *message)
478481
/* PyObject* parser_do_parse(PyObject* args, int type)
479482
*
480483
* Internal function to actually execute the parse and return the result if
481-
* successful, or set an exception if not.
484+
* successful or set an exception if not.
482485
*
483486
*/
484487
static PyObject*
@@ -494,10 +497,8 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type)
494497
(type == PyST_EXPR)
495498
? eval_input : file_input);
496499

497-
if (n != 0)
498-
res = parser_newstobject(n, type);
499-
else
500-
err_string("could not parse string");
500+
if (n)
501+
res = parser_newstobject(n, type);
501502
}
502503
return (res);
503504
}

‎Python/compile.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1241,6 +1241,9 @@ decode_utf8(char **sPtr, char *end, char* encoding)
12411241
#endif
12421242
}
12431243

1244+
/* compiler.transformer.Transformer.decode_literal depends on what
1245+
might seem like minor details of this function -- changes here
1246+
must be reflected there. */
12441247
static PyObject *
12451248
parsestr(struct compiling *c, char *s)
12461249
{

0 commit comments

Comments
 (0)