"""Parse tree transformation module. | |
Transforms Python source code into an abstract syntax tree (AST) | |
defined in the ast module. | |
The simplest ways to invoke this module are via parse and parseFile. | |
parse(buf) -> AST | |
parseFile(path) -> AST | |
""" | |
# Original version written by Greg Stein (gstein@lyra.org) | |
# and Bill Tutt (rassilon@lima.mudlib.org) | |
# February 1997. | |
# | |
# Modifications and improvements for Python 2.0 by Jeremy Hylton and | |
# Mark Hammond | |
# | |
# Some fixes to try to have correct line number on almost all nodes | |
# (except Module, Discard and Stmt) added by Sylvain Thenault | |
# | |
# Portions of this file are: | |
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. | |
# | |
# This module is provided under a BSD-ish license. See | |
# http://www.opensource.org/licenses/bsd-license.html | |
# and replace OWNER, ORGANIZATION, and YEAR as appropriate. | |
from compiler.ast import * | |
import parser | |
import symbol | |
import token | |
class WalkerError(StandardError): | |
pass | |
from compiler.consts import CO_VARARGS, CO_VARKEYWORDS | |
from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY | |
def parseFile(path): | |
f = open(path, "U") | |
# XXX The parser API tolerates files without a trailing newline, | |
# but not strings without a trailing newline. Always add an extra | |
# newline to the file contents, since we're going through the string | |
# version of the API. | |
src = f.read() + "\n" | |
f.close() | |
return parse(src) | |
def parse(buf, mode="exec"): | |
if mode == "exec" or mode == "single": | |
return Transformer().parsesuite(buf) | |
elif mode == "eval": | |
return Transformer().parseexpr(buf) | |
else: | |
raise ValueError("compile() arg 3 must be" | |
" 'exec' or 'eval' or 'single'") | |
def asList(nodes): | |
l = [] | |
for item in nodes: | |
if hasattr(item, "asList"): | |
l.append(item.asList()) | |
else: | |
if type(item) is type( (None, None) ): | |
l.append(tuple(asList(item))) | |
elif type(item) is type( [] ): | |
l.append(asList(item)) | |
else: | |
l.append(item) | |
return l | |
def extractLineNo(ast): | |
if not isinstance(ast[1], tuple): | |
# get a terminal node | |
return ast[2] | |
for child in ast[1:]: | |
if isinstance(child, tuple): | |
lineno = extractLineNo(child) | |
if lineno is not None: | |
return lineno | |
def Node(*args): | |
kind = args[0] | |
if kind in nodes: | |
try: | |
return nodes[kind](*args[1:]) | |
except TypeError: | |
print nodes[kind], len(args), args | |
raise | |
else: | |
raise WalkerError, "Can't find appropriate Node type: %s" % str(args) | |
#return apply(ast.Node, args) | |
class Transformer: | |
"""Utility object for transforming Python parse trees. | |
Exposes the following methods: | |
tree = transform(ast_tree) | |
tree = parsesuite(text) | |
tree = parseexpr(text) | |
tree = parsefile(fileob | filename) | |
""" | |
def __init__(self): | |
self._dispatch = {} | |
for value, name in symbol.sym_name.items(): | |
if hasattr(self, name): | |
self._dispatch[value] = getattr(self, name) | |
self._dispatch[token.NEWLINE] = self.com_NEWLINE | |
self._atom_dispatch = {token.LPAR: self.atom_lpar, | |
token.LSQB: self.atom_lsqb, | |
token.LBRACE: self.atom_lbrace, | |
token.BACKQUOTE: self.atom_backquote, | |
token.NUMBER: self.atom_number, | |
token.STRING: self.atom_string, | |
token.NAME: self.atom_name, | |
} | |
self.encoding = None | |
def transform(self, tree): | |
"""Transform an AST into a modified parse tree.""" | |
if not (isinstance(tree, tuple) or isinstance(tree, list)): | |
tree = parser.st2tuple(tree, line_info=1) | |
return self.compile_node(tree) | |
def parsesuite(self, text): | |
"""Return a modified parse tree for the given suite text.""" | |
return self.transform(parser.suite(text)) | |
def parseexpr(self, text): | |
"""Return a modified parse tree for the given expression text.""" | |
return self.transform(parser.expr(text)) | |
def parsefile(self, file): | |
"""Return a modified parse tree for the contents of the given file.""" | |
if type(file) == type(''): | |
file = open(file) | |
return self.parsesuite(file.read()) | |
# -------------------------------------------------------------- | |
# | |
# PRIVATE METHODS | |
# | |
def compile_node(self, node): | |
### emit a line-number node? | |
n = node[0] | |
if n == symbol.encoding_decl: | |
self.encoding = node[2] | |
node = node[1] | |
n = node[0] | |
if n == symbol.single_input: | |
return self.single_input(node[1:]) | |
if n == symbol.file_input: | |
return self.file_input(node[1:]) | |
if n == symbol.eval_input: | |
return self.eval_input(node[1:]) | |
if n == symbol.lambdef: | |
return self.lambdef(node[1:]) | |
if n == symbol.funcdef: | |
return self.funcdef(node[1:]) | |
if n == symbol.classdef: | |
return self.classdef(node[1:]) | |
raise WalkerError, ('unexpected node type', n) | |
def single_input(self, node): | |
### do we want to do anything about being "interactive" ? | |
# NEWLINE | simple_stmt | compound_stmt NEWLINE | |
n = node[0][0] | |
if n != token.NEWLINE: | |
return self.com_stmt(node[0]) | |
return Pass() | |
def file_input(self, nodelist): | |
doc = self.get_docstring(nodelist, symbol.file_input) | |
if doc is not None: | |
i = 1 | |
else: | |
i = 0 | |
stmts = [] | |
for node in nodelist[i:]: | |
if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: | |
self.com_append_stmt(stmts, node) | |
return Module(doc, Stmt(stmts)) | |
def eval_input(self, nodelist): | |
# from the built-in function input() | |
### is this sufficient? | |
return Expression(self.com_node(nodelist[0])) | |
def decorator_name(self, nodelist): | |
listlen = len(nodelist) | |
assert listlen >= 1 and listlen % 2 == 1 | |
item = self.atom_name(nodelist) | |
i = 1 | |
while i < listlen: | |
assert nodelist[i][0] == token.DOT | |
assert nodelist[i + 1][0] == token.NAME | |
item = Getattr(item, nodelist[i + 1][1]) | |
i += 2 | |
return item | |
def decorator(self, nodelist): | |
# '@' dotted_name [ '(' [arglist] ')' ] | |
assert len(nodelist) in (3, 5, 6) | |
assert nodelist[0][0] == token.AT | |
assert nodelist[-1][0] == token.NEWLINE | |
assert nodelist[1][0] == symbol.dotted_name | |
funcname = self.decorator_name(nodelist[1][1:]) | |
if len(nodelist) > 3: | |
assert nodelist[2][0] == token.LPAR | |
expr = self.com_call_function(funcname, nodelist[3]) | |
else: | |
expr = funcname | |
return expr | |
def decorators(self, nodelist): | |
# decorators: decorator ([NEWLINE] decorator)* NEWLINE | |
items = [] | |
for dec_nodelist in nodelist: | |
assert dec_nodelist[0] == symbol.decorator | |
items.append(self.decorator(dec_nodelist[1:])) | |
return Decorators(items) | |
def decorated(self, nodelist): | |
assert nodelist[0][0] == symbol.decorators | |
if nodelist[1][0] == symbol.funcdef: | |
n = [nodelist[0]] + list(nodelist[1][1:]) | |
return self.funcdef(n) | |
elif nodelist[1][0] == symbol.classdef: | |
decorators = self.decorators(nodelist[0][1:]) | |
cls = self.classdef(nodelist[1][1:]) | |
cls.decorators = decorators | |
return cls | |
raise WalkerError() | |
def funcdef(self, nodelist): | |
# -6 -5 -4 -3 -2 -1 | |
# funcdef: [decorators] 'def' NAME parameters ':' suite | |
# parameters: '(' [varargslist] ')' | |
if len(nodelist) == 6: | |
assert nodelist[0][0] == symbol.decorators | |
decorators = self.decorators(nodelist[0][1:]) | |
else: | |
assert len(nodelist) == 5 | |
decorators = None | |
lineno = nodelist[-4][2] | |
name = nodelist[-4][1] | |
args = nodelist[-3][2] | |
if args[0] == symbol.varargslist: | |
names, defaults, flags = self.com_arglist(args[1:]) | |
else: | |
names = defaults = () | |
flags = 0 | |
doc = self.get_docstring(nodelist[-1]) | |
# code for function | |
code = self.com_node(nodelist[-1]) | |
if doc is not None: | |
assert isinstance(code, Stmt) | |
assert isinstance(code.nodes[0], Discard) | |
del code.nodes[0] | |
return Function(decorators, name, names, defaults, flags, doc, code, | |
lineno=lineno) | |
def lambdef(self, nodelist): | |
# lambdef: 'lambda' [varargslist] ':' test | |
if nodelist[2][0] == symbol.varargslist: | |
names, defaults, flags = self.com_arglist(nodelist[2][1:]) | |
else: | |
names = defaults = () | |
flags = 0 | |
# code for lambda | |
code = self.com_node(nodelist[-1]) | |
return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) | |
old_lambdef = lambdef | |
def classdef(self, nodelist): | |
# classdef: 'class' NAME ['(' [testlist] ')'] ':' suite | |
name = nodelist[1][1] | |
doc = self.get_docstring(nodelist[-1]) | |
if nodelist[2][0] == token.COLON: | |
bases = [] | |
elif nodelist[3][0] == token.RPAR: | |
bases = [] | |
else: | |
bases = self.com_bases(nodelist[3]) | |
# code for class | |
code = self.com_node(nodelist[-1]) | |
if doc is not None: | |
assert isinstance(code, Stmt) | |
assert isinstance(code.nodes[0], Discard) | |
del code.nodes[0] | |
return Class(name, bases, doc, code, lineno=nodelist[1][2]) | |
def stmt(self, nodelist): | |
return self.com_stmt(nodelist[0]) | |
small_stmt = stmt | |
flow_stmt = stmt | |
compound_stmt = stmt | |
def simple_stmt(self, nodelist): | |
# small_stmt (';' small_stmt)* [';'] NEWLINE | |
stmts = [] | |
for i in range(0, len(nodelist), 2): | |
self.com_append_stmt(stmts, nodelist[i]) | |
return Stmt(stmts) | |
def parameters(self, nodelist): | |
raise WalkerError | |
def varargslist(self, nodelist): | |
raise WalkerError | |
def fpdef(self, nodelist): | |
raise WalkerError | |
def fplist(self, nodelist): | |
raise WalkerError | |
def dotted_name(self, nodelist): | |
raise WalkerError | |
def comp_op(self, nodelist): | |
raise WalkerError | |
def trailer(self, nodelist): | |
raise WalkerError | |
def sliceop(self, nodelist): | |
raise WalkerError | |
def argument(self, nodelist): | |
raise WalkerError | |
# -------------------------------------------------------------- | |
# | |
# STATEMENT NODES (invoked by com_node()) | |
# | |
def expr_stmt(self, nodelist): | |
# augassign testlist | testlist ('=' testlist)* | |
en = nodelist[-1] | |
exprNode = self.lookup_node(en)(en[1:]) | |
if len(nodelist) == 1: | |
return Discard(exprNode, lineno=exprNode.lineno) | |
if nodelist[1][0] == token.EQUAL: | |
nodesl = [] | |
for i in range(0, len(nodelist) - 2, 2): | |
nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) | |
return Assign(nodesl, exprNode, lineno=nodelist[1][2]) | |
else: | |
lval = self.com_augassign(nodelist[0]) | |
op = self.com_augassign_op(nodelist[1]) | |
return AugAssign(lval, op[1], exprNode, lineno=op[2]) | |
raise WalkerError, "can't get here" | |
def print_stmt(self, nodelist): | |
# print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) | |
items = [] | |
if len(nodelist) == 1: | |
start = 1 | |
dest = None | |
elif nodelist[1][0] == token.RIGHTSHIFT: | |
assert len(nodelist) == 3 \ | |
or nodelist[3][0] == token.COMMA | |
dest = self.com_node(nodelist[2]) | |
start = 4 | |
else: | |
dest = None | |
start = 1 | |
for i in range(start, len(nodelist), 2): | |
items.append(self.com_node(nodelist[i])) | |
if nodelist[-1][0] == token.COMMA: | |
return Print(items, dest, lineno=nodelist[0][2]) | |
return Printnl(items, dest, lineno=nodelist[0][2]) | |
def del_stmt(self, nodelist): | |
return self.com_assign(nodelist[1], OP_DELETE) | |
def pass_stmt(self, nodelist): | |
return Pass(lineno=nodelist[0][2]) | |
def break_stmt(self, nodelist): | |
return Break(lineno=nodelist[0][2]) | |
def continue_stmt(self, nodelist): | |
return Continue(lineno=nodelist[0][2]) | |
def return_stmt(self, nodelist): | |
# return: [testlist] | |
if len(nodelist) < 2: | |
return Return(Const(None), lineno=nodelist[0][2]) | |
return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) | |
def yield_stmt(self, nodelist): | |
expr = self.com_node(nodelist[0]) | |
return Discard(expr, lineno=expr.lineno) | |
def yield_expr(self, nodelist): | |
if len(nodelist) > 1: | |
value = self.com_node(nodelist[1]) | |
else: | |
value = Const(None) | |
return Yield(value, lineno=nodelist[0][2]) | |
def raise_stmt(self, nodelist): | |
# raise: [test [',' test [',' test]]] | |
if len(nodelist) > 5: | |
expr3 = self.com_node(nodelist[5]) | |
else: | |
expr3 = None | |
if len(nodelist) > 3: | |
expr2 = self.com_node(nodelist[3]) | |
else: | |
expr2 = None | |
if len(nodelist) > 1: | |
expr1 = self.com_node(nodelist[1]) | |
else: | |
expr1 = None | |
return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) | |
def import_stmt(self, nodelist): | |
# import_stmt: import_name | import_from | |
assert len(nodelist) == 1 | |
return self.com_node(nodelist[0]) | |
def import_name(self, nodelist): | |
# import_name: 'import' dotted_as_names | |
return Import(self.com_dotted_as_names(nodelist[1]), | |
lineno=nodelist[0][2]) | |
def import_from(self, nodelist): | |
# import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | | |
# '(' import_as_names ')' | import_as_names) | |
assert nodelist[0][1] == 'from' | |
idx = 1 | |
while nodelist[idx][1] == '.': | |
idx += 1 | |
level = idx - 1 | |
if nodelist[idx][0] == symbol.dotted_name: | |
fromname = self.com_dotted_name(nodelist[idx]) | |
idx += 1 | |
else: | |
fromname = "" | |
assert nodelist[idx][1] == 'import' | |
if nodelist[idx + 1][0] == token.STAR: | |
return From(fromname, [('*', None)], level, | |
lineno=nodelist[0][2]) | |
else: | |
node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)] | |
return From(fromname, self.com_import_as_names(node), level, | |
lineno=nodelist[0][2]) | |
def global_stmt(self, nodelist): | |
# global: NAME (',' NAME)* | |
names = [] | |
for i in range(1, len(nodelist), 2): | |
names.append(nodelist[i][1]) | |
return Global(names, lineno=nodelist[0][2]) | |
def exec_stmt(self, nodelist): | |
# exec_stmt: 'exec' expr ['in' expr [',' expr]] | |
expr1 = self.com_node(nodelist[1]) | |
if len(nodelist) >= 4: | |
expr2 = self.com_node(nodelist[3]) | |
if len(nodelist) >= 6: | |
expr3 = self.com_node(nodelist[5]) | |
else: | |
expr3 = None | |
else: | |
expr2 = expr3 = None | |
return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) | |
def assert_stmt(self, nodelist): | |
# 'assert': test, [',' test] | |
expr1 = self.com_node(nodelist[1]) | |
if (len(nodelist) == 4): | |
expr2 = self.com_node(nodelist[3]) | |
else: | |
expr2 = None | |
return Assert(expr1, expr2, lineno=nodelist[0][2]) | |
def if_stmt(self, nodelist): | |
# if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] | |
tests = [] | |
for i in range(0, len(nodelist) - 3, 4): | |
testNode = self.com_node(nodelist[i + 1]) | |
suiteNode = self.com_node(nodelist[i + 3]) | |
tests.append((testNode, suiteNode)) | |
if len(nodelist) % 4 == 3: | |
elseNode = self.com_node(nodelist[-1]) | |
## elseNode.lineno = nodelist[-1][1][2] | |
else: | |
elseNode = None | |
return If(tests, elseNode, lineno=nodelist[0][2]) | |
def while_stmt(self, nodelist): | |
# 'while' test ':' suite ['else' ':' suite] | |
testNode = self.com_node(nodelist[1]) | |
bodyNode = self.com_node(nodelist[3]) | |
if len(nodelist) > 4: | |
elseNode = self.com_node(nodelist[6]) | |
else: | |
elseNode = None | |
return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) | |
def for_stmt(self, nodelist): | |
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] | |
assignNode = self.com_assign(nodelist[1], OP_ASSIGN) | |
listNode = self.com_node(nodelist[3]) | |
bodyNode = self.com_node(nodelist[5]) | |
if len(nodelist) > 8: | |
elseNode = self.com_node(nodelist[8]) | |
else: | |
elseNode = None | |
return For(assignNode, listNode, bodyNode, elseNode, | |
lineno=nodelist[0][2]) | |
def try_stmt(self, nodelist): | |
return self.com_try_except_finally(nodelist) | |
def with_stmt(self, nodelist): | |
return self.com_with(nodelist) | |
def with_var(self, nodelist): | |
return self.com_with_var(nodelist) | |
def suite(self, nodelist): | |
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT | |
if len(nodelist) == 1: | |
return self.com_stmt(nodelist[0]) | |
stmts = [] | |
for node in nodelist: | |
if node[0] == symbol.stmt: | |
self.com_append_stmt(stmts, node) | |
return Stmt(stmts) | |
# -------------------------------------------------------------- | |
# | |
# EXPRESSION NODES (invoked by com_node()) | |
# | |
def testlist(self, nodelist): | |
# testlist: expr (',' expr)* [','] | |
# testlist_safe: test [(',' test)+ [',']] | |
# exprlist: expr (',' expr)* [','] | |
return self.com_binary(Tuple, nodelist) | |
testlist_safe = testlist # XXX | |
testlist1 = testlist | |
exprlist = testlist | |
def testlist_comp(self, nodelist): | |
# test ( comp_for | (',' test)* [','] ) | |
assert nodelist[0][0] == symbol.test | |
if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for: | |
test = self.com_node(nodelist[0]) | |
return self.com_generator_expression(test, nodelist[1]) | |
return self.testlist(nodelist) | |
def test(self, nodelist): | |
# or_test ['if' or_test 'else' test] | lambdef | |
if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: | |
return self.lambdef(nodelist[0]) | |
then = self.com_node(nodelist[0]) | |
if len(nodelist) > 1: | |
assert len(nodelist) == 5 | |
assert nodelist[1][1] == 'if' | |
assert nodelist[3][1] == 'else' | |
test = self.com_node(nodelist[2]) | |
else_ = self.com_node(nodelist[4]) | |
return IfExp(test, then, else_, lineno=nodelist[1][2]) | |
return then | |
def or_test(self, nodelist): | |
# and_test ('or' and_test)* | lambdef | |
if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: | |
return self.lambdef(nodelist[0]) | |
return self.com_binary(Or, nodelist) | |
old_test = or_test | |
def and_test(self, nodelist): | |
# not_test ('and' not_test)* | |
return self.com_binary(And, nodelist) | |
def not_test(self, nodelist): | |
# 'not' not_test | comparison | |
result = self.com_node(nodelist[-1]) | |
if len(nodelist) == 2: | |
return Not(result, lineno=nodelist[0][2]) | |
return result | |
def comparison(self, nodelist): | |
# comparison: expr (comp_op expr)* | |
node = self.com_node(nodelist[0]) | |
if len(nodelist) == 1: | |
return node | |
results = [] | |
for i in range(2, len(nodelist), 2): | |
nl = nodelist[i-1] | |
# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' | |
# | 'in' | 'not' 'in' | 'is' | 'is' 'not' | |
n = nl[1] | |
if n[0] == token.NAME: | |
type = n[1] | |
if len(nl) == 3: | |
if type == 'not': | |
type = 'not in' | |
else: | |
type = 'is not' | |
else: | |
type = _cmp_types[n[0]] | |
lineno = nl[1][2] | |
results.append((type, self.com_node(nodelist[i]))) | |
# we need a special "compare" node so that we can distinguish | |
# 3 < x < 5 from (3 < x) < 5 | |
# the two have very different semantics and results (note that the | |
# latter form is always true) | |
return Compare(node, results, lineno=lineno) | |
def expr(self, nodelist): | |
# xor_expr ('|' xor_expr)* | |
return self.com_binary(Bitor, nodelist) | |
def xor_expr(self, nodelist): | |
# xor_expr ('^' xor_expr)* | |
return self.com_binary(Bitxor, nodelist) | |
def and_expr(self, nodelist): | |
# xor_expr ('&' xor_expr)* | |
return self.com_binary(Bitand, nodelist) | |
def shift_expr(self, nodelist): | |
# shift_expr ('<<'|'>>' shift_expr)* | |
node = self.com_node(nodelist[0]) | |
for i in range(2, len(nodelist), 2): | |
right = self.com_node(nodelist[i]) | |
if nodelist[i-1][0] == token.LEFTSHIFT: | |
node = LeftShift([node, right], lineno=nodelist[1][2]) | |
elif nodelist[i-1][0] == token.RIGHTSHIFT: | |
node = RightShift([node, right], lineno=nodelist[1][2]) | |
else: | |
raise ValueError, "unexpected token: %s" % nodelist[i-1][0] | |
return node | |
def arith_expr(self, nodelist): | |
node = self.com_node(nodelist[0]) | |
for i in range(2, len(nodelist), 2): | |
right = self.com_node(nodelist[i]) | |
if nodelist[i-1][0] == token.PLUS: | |
node = Add([node, right], lineno=nodelist[1][2]) | |
elif nodelist[i-1][0] == token.MINUS: | |
node = Sub([node, right], lineno=nodelist[1][2]) | |
else: | |
raise ValueError, "unexpected token: %s" % nodelist[i-1][0] | |
return node | |
def term(self, nodelist): | |
node = self.com_node(nodelist[0]) | |
for i in range(2, len(nodelist), 2): | |
right = self.com_node(nodelist[i]) | |
t = nodelist[i-1][0] | |
if t == token.STAR: | |
node = Mul([node, right]) | |
elif t == token.SLASH: | |
node = Div([node, right]) | |
elif t == token.PERCENT: | |
node = Mod([node, right]) | |
elif t == token.DOUBLESLASH: | |
node = FloorDiv([node, right]) | |
else: | |
raise ValueError, "unexpected token: %s" % t | |
node.lineno = nodelist[1][2] | |
return node | |
def factor(self, nodelist): | |
elt = nodelist[0] | |
t = elt[0] | |
node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) | |
# need to handle (unary op)constant here... | |
if t == token.PLUS: | |
return UnaryAdd(node, lineno=elt[2]) | |
elif t == token.MINUS: | |
return UnarySub(node, lineno=elt[2]) | |
elif t == token.TILDE: | |
node = Invert(node, lineno=elt[2]) | |
return node | |
def power(self, nodelist): | |
# power: atom trailer* ('**' factor)* | |
node = self.com_node(nodelist[0]) | |
for i in range(1, len(nodelist)): | |
elt = nodelist[i] | |
if elt[0] == token.DOUBLESTAR: | |
return Power([node, self.com_node(nodelist[i+1])], | |
lineno=elt[2]) | |
node = self.com_apply_trailer(node, elt) | |
return node | |
def atom(self, nodelist): | |
return self._atom_dispatch[nodelist[0][0]](nodelist) | |
def atom_lpar(self, nodelist): | |
if nodelist[1][0] == token.RPAR: | |
return Tuple((), lineno=nodelist[0][2]) | |
return self.com_node(nodelist[1]) | |
def atom_lsqb(self, nodelist): | |
if nodelist[1][0] == token.RSQB: | |
return List((), lineno=nodelist[0][2]) | |
return self.com_list_constructor(nodelist[1]) | |
def atom_lbrace(self, nodelist): | |
if nodelist[1][0] == token.RBRACE: | |
return Dict((), lineno=nodelist[0][2]) | |
return self.com_dictorsetmaker(nodelist[1]) | |
def atom_backquote(self, nodelist): | |
return Backquote(self.com_node(nodelist[1])) | |
def atom_number(self, nodelist): | |
### need to verify this matches compile.c | |
k = eval(nodelist[0][1]) | |
return Const(k, lineno=nodelist[0][2]) | |
def decode_literal(self, lit): | |
if self.encoding: | |
# this is particularly fragile & a bit of a | |
# hack... changes in compile.c:parsestr and | |
# tokenizer.c must be reflected here. | |
if self.encoding not in ['utf-8', 'iso-8859-1']: | |
lit = unicode(lit, 'utf-8').encode(self.encoding) | |
return eval("# coding: %s\n%s" % (self.encoding, lit)) | |
else: | |
return eval(lit) | |
def atom_string(self, nodelist): | |
k = '' | |
for node in nodelist: | |
k += self.decode_literal(node[1]) | |
return Const(k, lineno=nodelist[0][2]) | |
def atom_name(self, nodelist): | |
return Name(nodelist[0][1], lineno=nodelist[0][2]) | |
# -------------------------------------------------------------- | |
# | |
# INTERNAL PARSING UTILITIES | |
# | |
# The use of com_node() introduces a lot of extra stack frames, | |
# enough to cause a stack overflow compiling test.test_parser with | |
# the standard interpreter recursionlimit. The com_node() is a | |
# convenience function that hides the dispatch details, but comes | |
# at a very high cost. It is more efficient to dispatch directly | |
# in the callers. In these cases, use lookup_node() and call the | |
# dispatched node directly. | |
def lookup_node(self, node): | |
return self._dispatch[node[0]] | |
def com_node(self, node): | |
# Note: compile.c has handling in com_node for del_stmt, pass_stmt, | |
# break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, | |
# and compound_stmt. | |
# We'll just dispatch them. | |
return self._dispatch[node[0]](node[1:]) | |
def com_NEWLINE(self, *args): | |
# A ';' at the end of a line can make a NEWLINE token appear | |
# here, Render it harmless. (genc discards ('discard', | |
# ('const', xxxx)) Nodes) | |
return Discard(Const(None)) | |
def com_arglist(self, nodelist): | |
# varargslist: | |
# (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | |
# | fpdef ['=' test] (',' fpdef ['=' test])* [','] | |
# fpdef: NAME | '(' fplist ')' | |
# fplist: fpdef (',' fpdef)* [','] | |
names = [] | |
defaults = [] | |
flags = 0 | |
i = 0 | |
while i < len(nodelist): | |
node = nodelist[i] | |
if node[0] == token.STAR or node[0] == token.DOUBLESTAR: | |
if node[0] == token.STAR: | |
node = nodelist[i+1] | |
if node[0] == token.NAME: | |
names.append(node[1]) | |
flags = flags | CO_VARARGS | |
i = i + 3 | |
if i < len(nodelist): | |
# should be DOUBLESTAR | |
t = nodelist[i][0] | |
if t == token.DOUBLESTAR: | |
node = nodelist[i+1] | |
else: | |
raise ValueError, "unexpected token: %s" % t | |
names.append(node[1]) | |
flags = flags | CO_VARKEYWORDS | |
break | |
# fpdef: NAME | '(' fplist ')' | |
names.append(self.com_fpdef(node)) | |
i = i + 1 | |
if i < len(nodelist) and nodelist[i][0] == token.EQUAL: | |
defaults.append(self.com_node(nodelist[i + 1])) | |
i = i + 2 | |
elif len(defaults): | |
# we have already seen an argument with default, but here | |
# came one without | |
raise SyntaxError, "non-default argument follows default argument" | |
# skip the comma | |
i = i + 1 | |
return names, defaults, flags | |
def com_fpdef(self, node): | |
# fpdef: NAME | '(' fplist ')' | |
if node[1][0] == token.LPAR: | |
return self.com_fplist(node[2]) | |
return node[1][1] | |
def com_fplist(self, node): | |
# fplist: fpdef (',' fpdef)* [','] | |
if len(node) == 2: | |
return self.com_fpdef(node[1]) | |
list = [] | |
for i in range(1, len(node), 2): | |
list.append(self.com_fpdef(node[i])) | |
return tuple(list) | |
def com_dotted_name(self, node): | |
# String together the dotted names and return the string | |
name = "" | |
for n in node: | |
if type(n) == type(()) and n[0] == 1: | |
name = name + n[1] + '.' | |
return name[:-1] | |
def com_dotted_as_name(self, node): | |
assert node[0] == symbol.dotted_as_name | |
node = node[1:] | |
dot = self.com_dotted_name(node[0][1:]) | |
if len(node) == 1: | |
return dot, None | |
assert node[1][1] == 'as' | |
assert node[2][0] == token.NAME | |
return dot, node[2][1] | |
def com_dotted_as_names(self, node): | |
assert node[0] == symbol.dotted_as_names | |
node = node[1:] | |
names = [self.com_dotted_as_name(node[0])] | |
for i in range(2, len(node), 2): | |
names.append(self.com_dotted_as_name(node[i])) | |
return names | |
def com_import_as_name(self, node): | |
assert node[0] == symbol.import_as_name | |
node = node[1:] | |
assert node[0][0] == token.NAME | |
if len(node) == 1: | |
return node[0][1], None | |
assert node[1][1] == 'as', node | |
assert node[2][0] == token.NAME | |
return node[0][1], node[2][1] | |
def com_import_as_names(self, node): | |
assert node[0] == symbol.import_as_names | |
node = node[1:] | |
names = [self.com_import_as_name(node[0])] | |
for i in range(2, len(node), 2): | |
names.append(self.com_import_as_name(node[i])) | |
return names | |
def com_bases(self, node): | |
bases = [] | |
for i in range(1, len(node), 2): | |
bases.append(self.com_node(node[i])) | |
return bases | |
def com_try_except_finally(self, nodelist): | |
# ('try' ':' suite | |
# ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] | |
# | 'finally' ':' suite)) | |
if nodelist[3][0] == token.NAME: | |
# first clause is a finally clause: only try-finally | |
return TryFinally(self.com_node(nodelist[2]), | |
self.com_node(nodelist[5]), | |
lineno=nodelist[0][2]) | |
#tryexcept: [TryNode, [except_clauses], elseNode)] | |
clauses = [] | |
elseNode = None | |
finallyNode = None | |
for i in range(3, len(nodelist), 3): | |
node = nodelist[i] | |
if node[0] == symbol.except_clause: | |
# except_clause: 'except' [expr [(',' | 'as') expr]] */ | |
if len(node) > 2: | |
expr1 = self.com_node(node[2]) | |
if len(node) > 4: | |
expr2 = self.com_assign(node[4], OP_ASSIGN) | |
else: | |
expr2 = None | |
else: | |
expr1 = expr2 = None | |
clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) | |
if node[0] == token.NAME: | |
if node[1] == 'else': | |
elseNode = self.com_node(nodelist[i+2]) | |
elif node[1] == 'finally': | |
finallyNode = self.com_node(nodelist[i+2]) | |
try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, | |
lineno=nodelist[0][2]) | |
if finallyNode: | |
return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) | |
else: | |
return try_except | |
def com_with(self, nodelist): | |
# with_stmt: 'with' with_item (',' with_item)* ':' suite | |
body = self.com_node(nodelist[-1]) | |
for i in range(len(nodelist) - 3, 0, -2): | |
ret = self.com_with_item(nodelist[i], body, nodelist[0][2]) | |
if i == 1: | |
return ret | |
body = ret | |
def com_with_item(self, nodelist, body, lineno): | |
# with_item: test ['as' expr] | |
if len(nodelist) == 4: | |
var = self.com_assign(nodelist[3], OP_ASSIGN) | |
else: | |
var = None | |
expr = self.com_node(nodelist[1]) | |
return With(expr, var, body, lineno=lineno) | |
def com_augassign_op(self, node): | |
assert node[0] == symbol.augassign | |
return node[1] | |
def com_augassign(self, node): | |
"""Return node suitable for lvalue of augmented assignment | |
Names, slices, and attributes are the only allowable nodes. | |
""" | |
l = self.com_node(node) | |
if l.__class__ in (Name, Slice, Subscript, Getattr): | |
return l | |
raise SyntaxError, "can't assign to %s" % l.__class__.__name__ | |
def com_assign(self, node, assigning): | |
# return a node suitable for use as an "lvalue" | |
# loop to avoid trivial recursion | |
while 1: | |
t = node[0] | |
if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp): | |
if len(node) > 2: | |
return self.com_assign_tuple(node, assigning) | |
node = node[1] | |
elif t in _assign_types: | |
if len(node) > 2: | |
raise SyntaxError, "can't assign to operator" | |
node = node[1] | |
elif t == symbol.power: | |
if node[1][0] != symbol.atom: | |
raise SyntaxError, "can't assign to operator" | |
if len(node) > 2: | |
primary = self.com_node(node[1]) | |
for i in range(2, len(node)-1): | |
ch = node[i] | |
if ch[0] == token.DOUBLESTAR: | |
raise SyntaxError, "can't assign to operator" | |
primary = self.com_apply_trailer(primary, ch) | |
return self.com_assign_trailer(primary, node[-1], | |
assigning) | |
node = node[1] | |
elif t == symbol.atom: | |
t = node[1][0] | |
if t == token.LPAR: | |
node = node[2] | |
if node[0] == token.RPAR: | |
raise SyntaxError, "can't assign to ()" | |
elif t == token.LSQB: | |
node = node[2] | |
if node[0] == token.RSQB: | |
raise SyntaxError, "can't assign to []" | |
return self.com_assign_list(node, assigning) | |
elif t == token.NAME: | |
return self.com_assign_name(node[1], assigning) | |
else: | |
raise SyntaxError, "can't assign to literal" | |
else: | |
raise SyntaxError, "bad assignment (%s)" % t | |
def com_assign_tuple(self, node, assigning): | |
assigns = [] | |
for i in range(1, len(node), 2): | |
assigns.append(self.com_assign(node[i], assigning)) | |
return AssTuple(assigns, lineno=extractLineNo(node)) | |
def com_assign_list(self, node, assigning): | |
assigns = [] | |
for i in range(1, len(node), 2): | |
if i + 1 < len(node): | |
if node[i + 1][0] == symbol.list_for: | |
raise SyntaxError, "can't assign to list comprehension" | |
assert node[i + 1][0] == token.COMMA, node[i + 1] | |
assigns.append(self.com_assign(node[i], assigning)) | |
return AssList(assigns, lineno=extractLineNo(node)) | |
def com_assign_name(self, node, assigning): | |
return AssName(node[1], assigning, lineno=node[2]) | |
def com_assign_trailer(self, primary, node, assigning): | |
t = node[1][0] | |
if t == token.DOT: | |
return self.com_assign_attr(primary, node[2], assigning) | |
if t == token.LSQB: | |
return self.com_subscriptlist(primary, node[2], assigning) | |
if t == token.LPAR: | |
raise SyntaxError, "can't assign to function call" | |
raise SyntaxError, "unknown trailer type: %s" % t | |
def com_assign_attr(self, primary, node, assigning): | |
return AssAttr(primary, node[1], assigning, lineno=node[-1]) | |
def com_binary(self, constructor, nodelist): | |
"Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." | |
l = len(nodelist) | |
if l == 1: | |
n = nodelist[0] | |
return self.lookup_node(n)(n[1:]) | |
items = [] | |
for i in range(0, l, 2): | |
n = nodelist[i] | |
items.append(self.lookup_node(n)(n[1:])) | |
return constructor(items, lineno=extractLineNo(nodelist)) | |
def com_stmt(self, node): | |
result = self.lookup_node(node)(node[1:]) | |
assert result is not None | |
if isinstance(result, Stmt): | |
return result | |
return Stmt([result]) | |
def com_append_stmt(self, stmts, node): | |
result = self.lookup_node(node)(node[1:]) | |
assert result is not None | |
if isinstance(result, Stmt): | |
stmts.extend(result.nodes) | |
else: | |
stmts.append(result) | |
def com_list_constructor(self, nodelist): | |
# listmaker: test ( list_for | (',' test)* [','] ) | |
values = [] | |
for i in range(1, len(nodelist)): | |
if nodelist[i][0] == symbol.list_for: | |
assert len(nodelist[i:]) == 1 | |
return self.com_list_comprehension(values[0], | |
nodelist[i]) | |
elif nodelist[i][0] == token.COMMA: | |
continue | |
values.append(self.com_node(nodelist[i])) | |
return List(values, lineno=values[0].lineno) | |
def com_list_comprehension(self, expr, node): | |
return self.com_comprehension(expr, None, node, 'list') | |
def com_comprehension(self, expr1, expr2, node, type): | |
# list_iter: list_for | list_if | |
# list_for: 'for' exprlist 'in' testlist [list_iter] | |
# list_if: 'if' test [list_iter] | |
# XXX should raise SyntaxError for assignment | |
# XXX(avassalotti) Set and dict comprehensions should have generator | |
# semantics. In other words, they shouldn't leak | |
# variables outside of the comprehension's scope. | |
lineno = node[1][2] | |
fors = [] | |
while node: | |
t = node[1][1] | |
if t == 'for': | |
assignNode = self.com_assign(node[2], OP_ASSIGN) | |
compNode = self.com_node(node[4]) | |
newfor = ListCompFor(assignNode, compNode, []) | |
newfor.lineno = node[1][2] | |
fors.append(newfor) | |
if len(node) == 5: | |
node = None | |
elif type == 'list': | |
node = self.com_list_iter(node[5]) | |
else: | |
node = self.com_comp_iter(node[5]) | |
elif t == 'if': | |
test = self.com_node(node[2]) | |
newif = ListCompIf(test, lineno=node[1][2]) | |
newfor.ifs.append(newif) | |
if len(node) == 3: | |
node = None | |
elif type == 'list': | |
node = self.com_list_iter(node[3]) | |
else: | |
node = self.com_comp_iter(node[3]) | |
else: | |
raise SyntaxError, \ | |
("unexpected comprehension element: %s %d" | |
% (node, lineno)) | |
if type == 'list': | |
return ListComp(expr1, fors, lineno=lineno) | |
elif type == 'set': | |
return SetComp(expr1, fors, lineno=lineno) | |
elif type == 'dict': | |
return DictComp(expr1, expr2, fors, lineno=lineno) | |
else: | |
raise ValueError("unexpected comprehension type: " + repr(type)) | |
def com_list_iter(self, node): | |
assert node[0] == symbol.list_iter | |
return node[1] | |
def com_comp_iter(self, node): | |
assert node[0] == symbol.comp_iter | |
return node[1] | |
def com_generator_expression(self, expr, node): | |
# comp_iter: comp_for | comp_if | |
# comp_for: 'for' exprlist 'in' test [comp_iter] | |
# comp_if: 'if' test [comp_iter] | |
lineno = node[1][2] | |
fors = [] | |
while node: | |
t = node[1][1] | |
if t == 'for': | |
assignNode = self.com_assign(node[2], OP_ASSIGN) | |
genNode = self.com_node(node[4]) | |
newfor = GenExprFor(assignNode, genNode, [], | |
lineno=node[1][2]) | |
fors.append(newfor) | |
if (len(node)) == 5: | |
node = None | |
else: | |
node = self.com_comp_iter(node[5]) | |
elif t == 'if': | |
test = self.com_node(node[2]) | |
newif = GenExprIf(test, lineno=node[1][2]) | |
newfor.ifs.append(newif) | |
if len(node) == 3: | |
node = None | |
else: | |
node = self.com_comp_iter(node[3]) | |
else: | |
raise SyntaxError, \ | |
("unexpected generator expression element: %s %d" | |
% (node, lineno)) | |
fors[0].is_outmost = True | |
return GenExpr(GenExprInner(expr, fors), lineno=lineno) | |
def com_dictorsetmaker(self, nodelist): | |
# dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | | |
# (test (comp_for | (',' test)* [','])) ) | |
assert nodelist[0] == symbol.dictorsetmaker | |
nodelist = nodelist[1:] | |
if len(nodelist) == 1 or nodelist[1][0] == token.COMMA: | |
# set literal | |
items = [] | |
for i in range(0, len(nodelist), 2): | |
items.append(self.com_node(nodelist[i])) | |
return Set(items, lineno=items[0].lineno) | |
elif nodelist[1][0] == symbol.comp_for: | |
# set comprehension | |
expr = self.com_node(nodelist[0]) | |
return self.com_comprehension(expr, None, nodelist[1], 'set') | |
elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for: | |
# dict comprehension | |
assert nodelist[1][0] == token.COLON | |
key = self.com_node(nodelist[0]) | |
value = self.com_node(nodelist[2]) | |
return self.com_comprehension(key, value, nodelist[3], 'dict') | |
else: | |
# dict literal | |
items = [] | |
for i in range(0, len(nodelist), 4): | |
items.append((self.com_node(nodelist[i]), | |
self.com_node(nodelist[i+2]))) | |
return Dict(items, lineno=items[0][0].lineno) | |
def com_apply_trailer(self, primaryNode, nodelist): | |
t = nodelist[1][0] | |
if t == token.LPAR: | |
return self.com_call_function(primaryNode, nodelist[2]) | |
if t == token.DOT: | |
return self.com_select_member(primaryNode, nodelist[2]) | |
if t == token.LSQB: | |
return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) | |
raise SyntaxError, 'unknown node type: %s' % t | |
def com_select_member(self, primaryNode, nodelist): | |
if nodelist[0] != token.NAME: | |
raise SyntaxError, "member must be a name" | |
return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) | |
def com_call_function(self, primaryNode, nodelist): | |
if nodelist[0] == token.RPAR: | |
return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) | |
args = [] | |
kw = 0 | |
star_node = dstar_node = None | |
len_nodelist = len(nodelist) | |
i = 1 | |
while i < len_nodelist: | |
node = nodelist[i] | |
if node[0]==token.STAR: | |
if star_node is not None: | |
raise SyntaxError, 'already have the varargs indentifier' | |
star_node = self.com_node(nodelist[i+1]) | |
i = i + 3 | |
continue | |
elif node[0]==token.DOUBLESTAR: | |
if dstar_node is not None: | |
raise SyntaxError, 'already have the kwargs indentifier' | |
dstar_node = self.com_node(nodelist[i+1]) | |
i = i + 3 | |
continue | |
# positional or named parameters | |
kw, result = self.com_argument(node, kw, star_node) | |
if len_nodelist != 2 and isinstance(result, GenExpr) \ | |
and len(node) == 3 and node[2][0] == symbol.comp_for: | |
# allow f(x for x in y), but reject f(x for x in y, 1) | |
# should use f((x for x in y), 1) instead of f(x for x in y, 1) | |
raise SyntaxError, 'generator expression needs parenthesis' | |
args.append(result) | |
i = i + 2 | |
return CallFunc(primaryNode, args, star_node, dstar_node, | |
lineno=extractLineNo(nodelist)) | |
def com_argument(self, nodelist, kw, star_node): | |
if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for: | |
test = self.com_node(nodelist[1]) | |
return 0, self.com_generator_expression(test, nodelist[2]) | |
if len(nodelist) == 2: | |
if kw: | |
raise SyntaxError, "non-keyword arg after keyword arg" | |
if star_node: | |
raise SyntaxError, "only named arguments may follow *expression" | |
return 0, self.com_node(nodelist[1]) | |
result = self.com_node(nodelist[3]) | |
n = nodelist[1] | |
while len(n) == 2 and n[0] != token.NAME: | |
n = n[1] | |
if n[0] != token.NAME: | |
raise SyntaxError, "keyword can't be an expression (%s)"%n[0] | |
node = Keyword(n[1], result, lineno=n[2]) | |
return 1, node | |
def com_subscriptlist(self, primary, nodelist, assigning): | |
# slicing: simple_slicing | extended_slicing | |
# simple_slicing: primary "[" short_slice "]" | |
# extended_slicing: primary "[" slice_list "]" | |
# slice_list: slice_item ("," slice_item)* [","] | |
# backwards compat slice for '[i:j]' | |
if len(nodelist) == 2: | |
sub = nodelist[1] | |
if (sub[1][0] == token.COLON or \ | |
(len(sub) > 2 and sub[2][0] == token.COLON)) and \ | |
sub[-1][0] != symbol.sliceop: | |
return self.com_slice(primary, sub, assigning) | |
subscripts = [] | |
for i in range(1, len(nodelist), 2): | |
subscripts.append(self.com_subscript(nodelist[i])) | |
return Subscript(primary, assigning, subscripts, | |
lineno=extractLineNo(nodelist)) | |
def com_subscript(self, node): | |
# slice_item: expression | proper_slice | ellipsis | |
ch = node[1] | |
t = ch[0] | |
if t == token.DOT and node[2][0] == token.DOT: | |
return Ellipsis() | |
if t == token.COLON or len(node) > 2: | |
return self.com_sliceobj(node) | |
return self.com_node(ch) | |
def com_sliceobj(self, node): | |
# proper_slice: short_slice | long_slice | |
# short_slice: [lower_bound] ":" [upper_bound] | |
# long_slice: short_slice ":" [stride] | |
# lower_bound: expression | |
# upper_bound: expression | |
# stride: expression | |
# | |
# Note: a stride may be further slicing... | |
items = [] | |
if node[1][0] == token.COLON: | |
items.append(Const(None)) | |
i = 2 | |
else: | |
items.append(self.com_node(node[1])) | |
# i == 2 is a COLON | |
i = 3 | |
if i < len(node) and node[i][0] == symbol.test: | |
items.append(self.com_node(node[i])) | |
i = i + 1 | |
else: | |
items.append(Const(None)) | |
# a short_slice has been built. look for long_slice now by looking | |
# for strides... | |
for j in range(i, len(node)): | |
ch = node[j] | |
if len(ch) == 2: | |
items.append(Const(None)) | |
else: | |
items.append(self.com_node(ch[2])) | |
return Sliceobj(items, lineno=extractLineNo(node)) | |
def com_slice(self, primary, node, assigning): | |
# short_slice: [lower_bound] ":" [upper_bound] | |
lower = upper = None | |
if len(node) == 3: | |
if node[1][0] == token.COLON: | |
upper = self.com_node(node[2]) | |
else: | |
lower = self.com_node(node[1]) | |
elif len(node) == 4: | |
lower = self.com_node(node[1]) | |
upper = self.com_node(node[3]) | |
return Slice(primary, assigning, lower, upper, | |
lineno=extractLineNo(node)) | |
def get_docstring(self, node, n=None): | |
if n is None: | |
n = node[0] | |
node = node[1:] | |
if n == symbol.suite: | |
if len(node) == 1: | |
return self.get_docstring(node[0]) | |
for sub in node: | |
if sub[0] == symbol.stmt: | |
return self.get_docstring(sub) | |
return None | |
if n == symbol.file_input: | |
for sub in node: | |
if sub[0] == symbol.stmt: | |
return self.get_docstring(sub) | |
return None | |
if n == symbol.atom: | |
if node[0][0] == token.STRING: | |
s = '' | |
for t in node: | |
s = s + eval(t[1]) | |
return s | |
return None | |
if n == symbol.stmt or n == symbol.simple_stmt \ | |
or n == symbol.small_stmt: | |
return self.get_docstring(node[0]) | |
if n in _doc_nodes and len(node) == 1: | |
return self.get_docstring(node[0]) | |
return None | |
_doc_nodes = [ | |
symbol.expr_stmt, | |
symbol.testlist, | |
symbol.testlist_safe, | |
symbol.test, | |
symbol.or_test, | |
symbol.and_test, | |
symbol.not_test, | |
symbol.comparison, | |
symbol.expr, | |
symbol.xor_expr, | |
symbol.and_expr, | |
symbol.shift_expr, | |
symbol.arith_expr, | |
symbol.term, | |
symbol.factor, | |
symbol.power, | |
] | |
# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' | |
# | 'in' | 'not' 'in' | 'is' | 'is' 'not' | |
_cmp_types = { | |
token.LESS : '<', | |
token.GREATER : '>', | |
token.EQEQUAL : '==', | |
token.EQUAL : '==', | |
token.LESSEQUAL : '<=', | |
token.GREATEREQUAL : '>=', | |
token.NOTEQUAL : '!=', | |
} | |
_legal_node_types = [ | |
symbol.funcdef, | |
symbol.classdef, | |
symbol.stmt, | |
symbol.small_stmt, | |
symbol.flow_stmt, | |
symbol.simple_stmt, | |
symbol.compound_stmt, | |
symbol.expr_stmt, | |
symbol.print_stmt, | |
symbol.del_stmt, | |
symbol.pass_stmt, | |
symbol.break_stmt, | |
symbol.continue_stmt, | |
symbol.return_stmt, | |
symbol.raise_stmt, | |
symbol.import_stmt, | |
symbol.global_stmt, | |
symbol.exec_stmt, | |
symbol.assert_stmt, | |
symbol.if_stmt, | |
symbol.while_stmt, | |
symbol.for_stmt, | |
symbol.try_stmt, | |
symbol.with_stmt, | |
symbol.suite, | |
symbol.testlist, | |
symbol.testlist_safe, | |
symbol.test, | |
symbol.and_test, | |
symbol.not_test, | |
symbol.comparison, | |
symbol.exprlist, | |
symbol.expr, | |
symbol.xor_expr, | |
symbol.and_expr, | |
symbol.shift_expr, | |
symbol.arith_expr, | |
symbol.term, | |
symbol.factor, | |
symbol.power, | |
symbol.atom, | |
] | |
if hasattr(symbol, 'yield_stmt'): | |
_legal_node_types.append(symbol.yield_stmt) | |
if hasattr(symbol, 'yield_expr'): | |
_legal_node_types.append(symbol.yield_expr) | |
_assign_types = [ | |
symbol.test, | |
symbol.or_test, | |
symbol.and_test, | |
symbol.not_test, | |
symbol.comparison, | |
symbol.expr, | |
symbol.xor_expr, | |
symbol.and_expr, | |
symbol.shift_expr, | |
symbol.arith_expr, | |
symbol.term, | |
symbol.factor, | |
] | |
_names = {} | |
for k, v in symbol.sym_name.items(): | |
_names[k] = v | |
for k, v in token.tok_name.items(): | |
_names[k] = v | |
def debug_tree(tree): | |
l = [] | |
for elt in tree: | |
if isinstance(elt, int): | |
l.append(_names.get(elt, elt)) | |
elif isinstance(elt, str): | |
l.append(elt) | |
else: | |
l.append(debug_tree(elt)) | |
return l |