summaryrefslogtreecommitdiff
path: root/lib/python2.7/compiler
diff options
context:
space:
mode:
authorrahulp132020-03-17 14:55:41 +0530
committerrahulp132020-03-17 14:55:41 +0530
commit296443137f4288cb030e92859ccfbe3204bc1088 (patch)
treeca4798c2da1e7244edc3bc108d81b462b537aea2 /lib/python2.7/compiler
parent0db48f6533517ecebfd9f0693f89deca28408b76 (diff)
downloadKiCad-eSim-296443137f4288cb030e92859ccfbe3204bc1088.tar.gz
KiCad-eSim-296443137f4288cb030e92859ccfbe3204bc1088.tar.bz2
KiCad-eSim-296443137f4288cb030e92859ccfbe3204bc1088.zip
initial commit
Diffstat (limited to 'lib/python2.7/compiler')
-rw-r--r--lib/python2.7/compiler/__init__.py31
-rw-r--r--lib/python2.7/compiler/ast.py1419
-rw-r--r--lib/python2.7/compiler/consts.py23
-rw-r--r--lib/python2.7/compiler/future.py74
-rw-r--r--lib/python2.7/compiler/misc.py73
-rw-r--r--lib/python2.7/compiler/pyassem.py763
-rw-r--r--lib/python2.7/compiler/pycodegen.py1555
-rw-r--r--lib/python2.7/compiler/symbols.py462
-rw-r--r--lib/python2.7/compiler/syntax.py46
-rw-r--r--lib/python2.7/compiler/transformer.py1535
-rw-r--r--lib/python2.7/compiler/visitor.py113
11 files changed, 6094 insertions, 0 deletions
diff --git a/lib/python2.7/compiler/__init__.py b/lib/python2.7/compiler/__init__.py
new file mode 100644
index 0000000..2a6f64f
--- /dev/null
+++ b/lib/python2.7/compiler/__init__.py
@@ -0,0 +1,31 @@
+"""Package for parsing and compiling Python source code
+
+There are several functions defined at the top level that are imported
+from modules contained in the package.
+
+parse(buf, mode="exec") -> AST
+ Converts a string containing Python source code to an abstract
+ syntax tree (AST). The AST is defined in compiler.ast.
+
+parseFile(path) -> AST
+ The same as parse(open(path))
+
+walk(ast, visitor, verbose=None)
+ Does a pre-order walk over the ast using the visitor instance.
+ See compiler.visitor for details.
+
+compile(source, filename, mode, flags=None, dont_inherit=None)
+ Returns a code object. A replacement for the builtin compile() function.
+
+compileFile(filename)
+ Generates a .pyc file by compiling filename.
+"""
+
+import warnings
+
+warnings.warn("The compiler package is deprecated and removed in Python 3.x.",
+ DeprecationWarning, stacklevel=2)
+
+from compiler.transformer import parse, parseFile
+from compiler.visitor import walk
+from compiler.pycodegen import compile, compileFile
diff --git a/lib/python2.7/compiler/ast.py b/lib/python2.7/compiler/ast.py
new file mode 100644
index 0000000..4c3fc16
--- /dev/null
+++ b/lib/python2.7/compiler/ast.py
@@ -0,0 +1,1419 @@
+"""Python abstract syntax node definitions
+
+This file is automatically generated by Tools/compiler/astgen.py
+"""
+from compiler.consts import CO_VARARGS, CO_VARKEYWORDS
+
+def flatten(seq):
+ l = []
+ for elt in seq:
+ t = type(elt)
+ if t is tuple or t is list:
+ for elt2 in flatten(elt):
+ l.append(elt2)
+ else:
+ l.append(elt)
+ return l
+
+def flatten_nodes(seq):
+ return [n for n in flatten(seq) if isinstance(n, Node)]
+
+nodes = {}
+
+class Node:
+ """Abstract base class for ast nodes."""
+ def getChildren(self):
+ pass # implemented by subclasses
+ def __iter__(self):
+ for n in self.getChildren():
+ yield n
+ def asList(self): # for backwards compatibility
+ return self.getChildren()
+ def getChildNodes(self):
+ pass # implemented by subclasses
+
+class EmptyNode(Node):
+ pass
+
+class Expression(Node):
+ # Expression is an artificial node class to support "eval"
+ nodes["expression"] = "Expression"
+ def __init__(self, node):
+ self.node = node
+
+ def getChildren(self):
+ return self.node,
+
+ def getChildNodes(self):
+ return self.node,
+
+ def __repr__(self):
+ return "Expression(%s)" % (repr(self.node))
+
+class Add(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "Add((%s, %s))" % (repr(self.left), repr(self.right))
+
+class And(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "And(%s)" % (repr(self.nodes),)
+
+class AssAttr(Node):
+ def __init__(self, expr, attrname, flags, lineno=None):
+ self.expr = expr
+ self.attrname = attrname
+ self.flags = flags
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr, self.attrname, self.flags
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "AssAttr(%s, %s, %s)" % (repr(self.expr), repr(self.attrname), repr(self.flags))
+
+class AssList(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "AssList(%s)" % (repr(self.nodes),)
+
+class AssName(Node):
+ def __init__(self, name, flags, lineno=None):
+ self.name = name
+ self.flags = flags
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.name, self.flags
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "AssName(%s, %s)" % (repr(self.name), repr(self.flags))
+
+class AssTuple(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "AssTuple(%s)" % (repr(self.nodes),)
+
+class Assert(Node):
+ def __init__(self, test, fail, lineno=None):
+ self.test = test
+ self.fail = fail
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.test)
+ children.append(self.fail)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.test)
+ if self.fail is not None:
+ nodelist.append(self.fail)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Assert(%s, %s)" % (repr(self.test), repr(self.fail))
+
+class Assign(Node):
+ def __init__(self, nodes, expr, lineno=None):
+ self.nodes = nodes
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.extend(flatten(self.nodes))
+ children.append(self.expr)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ nodelist.append(self.expr)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Assign(%s, %s)" % (repr(self.nodes), repr(self.expr))
+
+class AugAssign(Node):
+ def __init__(self, node, op, expr, lineno=None):
+ self.node = node
+ self.op = op
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.node, self.op, self.expr
+
+ def getChildNodes(self):
+ return self.node, self.expr
+
+ def __repr__(self):
+ return "AugAssign(%s, %s, %s)" % (repr(self.node), repr(self.op), repr(self.expr))
+
+class Backquote(Node):
+ def __init__(self, expr, lineno=None):
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr,
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "Backquote(%s)" % (repr(self.expr),)
+
+class Bitand(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Bitand(%s)" % (repr(self.nodes),)
+
+class Bitor(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Bitor(%s)" % (repr(self.nodes),)
+
+class Bitxor(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Bitxor(%s)" % (repr(self.nodes),)
+
+class Break(Node):
+ def __init__(self, lineno=None):
+ self.lineno = lineno
+
+ def getChildren(self):
+ return ()
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Break()"
+
+class CallFunc(Node):
+ def __init__(self, node, args, star_args = None, dstar_args = None, lineno=None):
+ self.node = node
+ self.args = args
+ self.star_args = star_args
+ self.dstar_args = dstar_args
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.node)
+ children.extend(flatten(self.args))
+ children.append(self.star_args)
+ children.append(self.dstar_args)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.node)
+ nodelist.extend(flatten_nodes(self.args))
+ if self.star_args is not None:
+ nodelist.append(self.star_args)
+ if self.dstar_args is not None:
+ nodelist.append(self.dstar_args)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "CallFunc(%s, %s, %s, %s)" % (repr(self.node), repr(self.args), repr(self.star_args), repr(self.dstar_args))
+
+class Class(Node):
+ def __init__(self, name, bases, doc, code, decorators = None, lineno=None):
+ self.name = name
+ self.bases = bases
+ self.doc = doc
+ self.code = code
+ self.decorators = decorators
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.name)
+ children.extend(flatten(self.bases))
+ children.append(self.doc)
+ children.append(self.code)
+ children.append(self.decorators)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.bases))
+ nodelist.append(self.code)
+ if self.decorators is not None:
+ nodelist.append(self.decorators)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Class(%s, %s, %s, %s, %s)" % (repr(self.name), repr(self.bases), repr(self.doc), repr(self.code), repr(self.decorators))
+
+class Compare(Node):
+ def __init__(self, expr, ops, lineno=None):
+ self.expr = expr
+ self.ops = ops
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.extend(flatten(self.ops))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ nodelist.extend(flatten_nodes(self.ops))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Compare(%s, %s)" % (repr(self.expr), repr(self.ops))
+
+class Const(Node):
+ def __init__(self, value, lineno=None):
+ self.value = value
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.value,
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Const(%s)" % (repr(self.value),)
+
+class Continue(Node):
+ def __init__(self, lineno=None):
+ self.lineno = lineno
+
+ def getChildren(self):
+ return ()
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Continue()"
+
+class Decorators(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Decorators(%s)" % (repr(self.nodes),)
+
+class Dict(Node):
+ def __init__(self, items, lineno=None):
+ self.items = items
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.items))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.items))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Dict(%s)" % (repr(self.items),)
+
+class Discard(Node):
+ def __init__(self, expr, lineno=None):
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr,
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "Discard(%s)" % (repr(self.expr),)
+
+class Div(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "Div((%s, %s))" % (repr(self.left), repr(self.right))
+
+class Ellipsis(Node):
+ def __init__(self, lineno=None):
+ self.lineno = lineno
+
+ def getChildren(self):
+ return ()
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Ellipsis()"
+
+class Exec(Node):
+ def __init__(self, expr, locals, globals, lineno=None):
+ self.expr = expr
+ self.locals = locals
+ self.globals = globals
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.append(self.locals)
+ children.append(self.globals)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ if self.locals is not None:
+ nodelist.append(self.locals)
+ if self.globals is not None:
+ nodelist.append(self.globals)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Exec(%s, %s, %s)" % (repr(self.expr), repr(self.locals), repr(self.globals))
+
+class FloorDiv(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "FloorDiv((%s, %s))" % (repr(self.left), repr(self.right))
+
+class For(Node):
+ def __init__(self, assign, list, body, else_, lineno=None):
+ self.assign = assign
+ self.list = list
+ self.body = body
+ self.else_ = else_
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.assign)
+ children.append(self.list)
+ children.append(self.body)
+ children.append(self.else_)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.assign)
+ nodelist.append(self.list)
+ nodelist.append(self.body)
+ if self.else_ is not None:
+ nodelist.append(self.else_)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "For(%s, %s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.body), repr(self.else_))
+
+class From(Node):
+ def __init__(self, modname, names, level, lineno=None):
+ self.modname = modname
+ self.names = names
+ self.level = level
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.modname, self.names, self.level
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "From(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level))
+
+class Function(Node):
+ def __init__(self, decorators, name, argnames, defaults, flags, doc, code, lineno=None):
+ self.decorators = decorators
+ self.name = name
+ self.argnames = argnames
+ self.defaults = defaults
+ self.flags = flags
+ self.doc = doc
+ self.code = code
+ self.lineno = lineno
+ self.varargs = self.kwargs = None
+ if flags & CO_VARARGS:
+ self.varargs = 1
+ if flags & CO_VARKEYWORDS:
+ self.kwargs = 1
+
+
+ def getChildren(self):
+ children = []
+ children.append(self.decorators)
+ children.append(self.name)
+ children.append(self.argnames)
+ children.extend(flatten(self.defaults))
+ children.append(self.flags)
+ children.append(self.doc)
+ children.append(self.code)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ if self.decorators is not None:
+ nodelist.append(self.decorators)
+ nodelist.extend(flatten_nodes(self.defaults))
+ nodelist.append(self.code)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Function(%s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.argnames), repr(self.defaults), repr(self.flags), repr(self.doc), repr(self.code))
+
+class GenExpr(Node):
+ def __init__(self, code, lineno=None):
+ self.code = code
+ self.lineno = lineno
+ self.argnames = ['.0']
+ self.varargs = self.kwargs = None
+
+
+ def getChildren(self):
+ return self.code,
+
+ def getChildNodes(self):
+ return self.code,
+
+ def __repr__(self):
+ return "GenExpr(%s)" % (repr(self.code),)
+
+class GenExprFor(Node):
+ def __init__(self, assign, iter, ifs, lineno=None):
+ self.assign = assign
+ self.iter = iter
+ self.ifs = ifs
+ self.lineno = lineno
+ self.is_outmost = False
+
+ def getChildren(self):
+ children = []
+ children.append(self.assign)
+ children.append(self.iter)
+ children.extend(flatten(self.ifs))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.assign)
+ nodelist.append(self.iter)
+ nodelist.extend(flatten_nodes(self.ifs))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenExprFor(%s, %s, %s)" % (repr(self.assign), repr(self.iter), repr(self.ifs))
+
+class GenExprIf(Node):
+ def __init__(self, test, lineno=None):
+ self.test = test
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.test,
+
+ def getChildNodes(self):
+ return self.test,
+
+ def __repr__(self):
+ return "GenExprIf(%s)" % (repr(self.test),)
+
+class GenExprInner(Node):
+ def __init__(self, expr, quals, lineno=None):
+ self.expr = expr
+ self.quals = quals
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.extend(flatten(self.quals))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ nodelist.extend(flatten_nodes(self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenExprInner(%s, %s)" % (repr(self.expr), repr(self.quals))
+
+class Getattr(Node):
+ def __init__(self, expr, attrname, lineno=None):
+ self.expr = expr
+ self.attrname = attrname
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr, self.attrname
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "Getattr(%s, %s)" % (repr(self.expr), repr(self.attrname))
+
+class Global(Node):
+ def __init__(self, names, lineno=None):
+ self.names = names
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.names,
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Global(%s)" % (repr(self.names),)
+
+class If(Node):
+ def __init__(self, tests, else_, lineno=None):
+ self.tests = tests
+ self.else_ = else_
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.extend(flatten(self.tests))
+ children.append(self.else_)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.tests))
+ if self.else_ is not None:
+ nodelist.append(self.else_)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "If(%s, %s)" % (repr(self.tests), repr(self.else_))
+
+class IfExp(Node):
+ def __init__(self, test, then, else_, lineno=None):
+ self.test = test
+ self.then = then
+ self.else_ = else_
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.test, self.then, self.else_
+
+ def getChildNodes(self):
+ return self.test, self.then, self.else_
+
+ def __repr__(self):
+ return "IfExp(%s, %s, %s)" % (repr(self.test), repr(self.then), repr(self.else_))
+
+class Import(Node):
+ def __init__(self, names, lineno=None):
+ self.names = names
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.names,
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Import(%s)" % (repr(self.names),)
+
+class Invert(Node):
+ def __init__(self, expr, lineno=None):
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr,
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "Invert(%s)" % (repr(self.expr),)
+
+class Keyword(Node):
+ def __init__(self, name, expr, lineno=None):
+ self.name = name
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.name, self.expr
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "Keyword(%s, %s)" % (repr(self.name), repr(self.expr))
+
+class Lambda(Node):
+ def __init__(self, argnames, defaults, flags, code, lineno=None):
+ self.argnames = argnames
+ self.defaults = defaults
+ self.flags = flags
+ self.code = code
+ self.lineno = lineno
+ self.varargs = self.kwargs = None
+ if flags & CO_VARARGS:
+ self.varargs = 1
+ if flags & CO_VARKEYWORDS:
+ self.kwargs = 1
+
+
+ def getChildren(self):
+ children = []
+ children.append(self.argnames)
+ children.extend(flatten(self.defaults))
+ children.append(self.flags)
+ children.append(self.code)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.defaults))
+ nodelist.append(self.code)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Lambda(%s, %s, %s, %s)" % (repr(self.argnames), repr(self.defaults), repr(self.flags), repr(self.code))
+
+class LeftShift(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "LeftShift((%s, %s))" % (repr(self.left), repr(self.right))
+
+class List(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "List(%s)" % (repr(self.nodes),)
+
+class ListComp(Node):
+ def __init__(self, expr, quals, lineno=None):
+ self.expr = expr
+ self.quals = quals
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.extend(flatten(self.quals))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ nodelist.extend(flatten_nodes(self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "ListComp(%s, %s)" % (repr(self.expr), repr(self.quals))
+
+class ListCompFor(Node):
+ def __init__(self, assign, list, ifs, lineno=None):
+ self.assign = assign
+ self.list = list
+ self.ifs = ifs
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.assign)
+ children.append(self.list)
+ children.extend(flatten(self.ifs))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.assign)
+ nodelist.append(self.list)
+ nodelist.extend(flatten_nodes(self.ifs))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "ListCompFor(%s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.ifs))
+
+class ListCompIf(Node):
+ def __init__(self, test, lineno=None):
+ self.test = test
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.test,
+
+ def getChildNodes(self):
+ return self.test,
+
+ def __repr__(self):
+ return "ListCompIf(%s)" % (repr(self.test),)
+
+class SetComp(Node):
+ def __init__(self, expr, quals, lineno=None):
+ self.expr = expr
+ self.quals = quals
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.extend(flatten(self.quals))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ nodelist.extend(flatten_nodes(self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "SetComp(%s, %s)" % (repr(self.expr), repr(self.quals))
+
+class DictComp(Node):
+ def __init__(self, key, value, quals, lineno=None):
+ self.key = key
+ self.value = value
+ self.quals = quals
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.key)
+ children.append(self.value)
+ children.extend(flatten(self.quals))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.key)
+ nodelist.append(self.value)
+ nodelist.extend(flatten_nodes(self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "DictComp(%s, %s, %s)" % (repr(self.key), repr(self.value), repr(self.quals))
+
+class Mod(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "Mod((%s, %s))" % (repr(self.left), repr(self.right))
+
+class Module(Node):
+ def __init__(self, doc, node, lineno=None):
+ self.doc = doc
+ self.node = node
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.doc, self.node
+
+ def getChildNodes(self):
+ return self.node,
+
+ def __repr__(self):
+ return "Module(%s, %s)" % (repr(self.doc), repr(self.node))
+
+class Mul(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "Mul((%s, %s))" % (repr(self.left), repr(self.right))
+
+class Name(Node):
+ def __init__(self, name, lineno=None):
+ self.name = name
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.name,
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Name(%s)" % (repr(self.name),)
+
+class Not(Node):
+ def __init__(self, expr, lineno=None):
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr,
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "Not(%s)" % (repr(self.expr),)
+
+class Or(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Or(%s)" % (repr(self.nodes),)
+
+class Pass(Node):
+ def __init__(self, lineno=None):
+ self.lineno = lineno
+
+ def getChildren(self):
+ return ()
+
+ def getChildNodes(self):
+ return ()
+
+ def __repr__(self):
+ return "Pass()"
+
+class Power(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "Power((%s, %s))" % (repr(self.left), repr(self.right))
+
+class Print(Node):
+ def __init__(self, nodes, dest, lineno=None):
+ self.nodes = nodes
+ self.dest = dest
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.extend(flatten(self.nodes))
+ children.append(self.dest)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ if self.dest is not None:
+ nodelist.append(self.dest)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Print(%s, %s)" % (repr(self.nodes), repr(self.dest))
+
+class Printnl(Node):
+ def __init__(self, nodes, dest, lineno=None):
+ self.nodes = nodes
+ self.dest = dest
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.extend(flatten(self.nodes))
+ children.append(self.dest)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ if self.dest is not None:
+ nodelist.append(self.dest)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Printnl(%s, %s)" % (repr(self.nodes), repr(self.dest))
+
+class Raise(Node):
+ def __init__(self, expr1, expr2, expr3, lineno=None):
+ self.expr1 = expr1
+ self.expr2 = expr2
+ self.expr3 = expr3
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr1)
+ children.append(self.expr2)
+ children.append(self.expr3)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ if self.expr1 is not None:
+ nodelist.append(self.expr1)
+ if self.expr2 is not None:
+ nodelist.append(self.expr2)
+ if self.expr3 is not None:
+ nodelist.append(self.expr3)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Raise(%s, %s, %s)" % (repr(self.expr1), repr(self.expr2), repr(self.expr3))
+
+class Return(Node):
+ def __init__(self, value, lineno=None):
+ self.value = value
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.value,
+
+ def getChildNodes(self):
+ return self.value,
+
+ def __repr__(self):
+ return "Return(%s)" % (repr(self.value),)
+
+class RightShift(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "RightShift((%s, %s))" % (repr(self.left), repr(self.right))
+
+class Set(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Set(%s)" % (repr(self.nodes),)
+
+class Slice(Node):
+ def __init__(self, expr, flags, lower, upper, lineno=None):
+ self.expr = expr
+ self.flags = flags
+ self.lower = lower
+ self.upper = upper
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.append(self.flags)
+ children.append(self.lower)
+ children.append(self.upper)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ if self.lower is not None:
+ nodelist.append(self.lower)
+ if self.upper is not None:
+ nodelist.append(self.upper)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Slice(%s, %s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.lower), repr(self.upper))
+
+class Sliceobj(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Sliceobj(%s)" % (repr(self.nodes),)
+
+class Stmt(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Stmt(%s)" % (repr(self.nodes),)
+
+class Sub(Node):
+ def __init__(self, leftright, lineno=None):
+ self.left = leftright[0]
+ self.right = leftright[1]
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.left, self.right
+
+ def getChildNodes(self):
+ return self.left, self.right
+
+ def __repr__(self):
+ return "Sub((%s, %s))" % (repr(self.left), repr(self.right))
+
+class Subscript(Node):
+ def __init__(self, expr, flags, subs, lineno=None):
+ self.expr = expr
+ self.flags = flags
+ self.subs = subs
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.append(self.flags)
+ children.extend(flatten(self.subs))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ nodelist.extend(flatten_nodes(self.subs))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Subscript(%s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.subs))
+
+class TryExcept(Node):
+ def __init__(self, body, handlers, else_, lineno=None):
+ self.body = body
+ self.handlers = handlers
+ self.else_ = else_
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.body)
+ children.extend(flatten(self.handlers))
+ children.append(self.else_)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.body)
+ nodelist.extend(flatten_nodes(self.handlers))
+ if self.else_ is not None:
+ nodelist.append(self.else_)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "TryExcept(%s, %s, %s)" % (repr(self.body), repr(self.handlers), repr(self.else_))
+
+class TryFinally(Node):
+ def __init__(self, body, final, lineno=None):
+ self.body = body
+ self.final = final
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.body, self.final
+
+ def getChildNodes(self):
+ return self.body, self.final
+
+ def __repr__(self):
+ return "TryFinally(%s, %s)" % (repr(self.body), repr(self.final))
+
+class Tuple(Node):
+ def __init__(self, nodes, lineno=None):
+ self.nodes = nodes
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.nodes))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "Tuple(%s)" % (repr(self.nodes),)
+
+class UnaryAdd(Node):
+ def __init__(self, expr, lineno=None):
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr,
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "UnaryAdd(%s)" % (repr(self.expr),)
+
+class UnarySub(Node):
+ def __init__(self, expr, lineno=None):
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.expr,
+
+ def getChildNodes(self):
+ return self.expr,
+
+ def __repr__(self):
+ return "UnarySub(%s)" % (repr(self.expr),)
+
+class While(Node):
+ def __init__(self, test, body, else_, lineno=None):
+ self.test = test
+ self.body = body
+ self.else_ = else_
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.test)
+ children.append(self.body)
+ children.append(self.else_)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.test)
+ nodelist.append(self.body)
+ if self.else_ is not None:
+ nodelist.append(self.else_)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "While(%s, %s, %s)" % (repr(self.test), repr(self.body), repr(self.else_))
+
+class With(Node):
+ def __init__(self, expr, vars, body, lineno=None):
+ self.expr = expr
+ self.vars = vars
+ self.body = body
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.append(self.vars)
+ children.append(self.body)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ if self.vars is not None:
+ nodelist.append(self.vars)
+ nodelist.append(self.body)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "With(%s, %s, %s)" % (repr(self.expr), repr(self.vars), repr(self.body))
+
+class Yield(Node):
+ def __init__(self, value, lineno=None):
+ self.value = value
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.value,
+
+ def getChildNodes(self):
+ return self.value,
+
+ def __repr__(self):
+ return "Yield(%s)" % (repr(self.value),)
+
+for name, obj in globals().items():
+ if isinstance(obj, type) and issubclass(obj, Node):
+ nodes[name.lower()] = obj
diff --git a/lib/python2.7/compiler/consts.py b/lib/python2.7/compiler/consts.py
new file mode 100644
index 0000000..c60b1d0
--- /dev/null
+++ b/lib/python2.7/compiler/consts.py
@@ -0,0 +1,23 @@
+# operation flags
+OP_ASSIGN = 'OP_ASSIGN'
+OP_DELETE = 'OP_DELETE'
+OP_APPLY = 'OP_APPLY'
+
+SC_LOCAL = 1
+SC_GLOBAL_IMPLICIT = 2
+SC_GLOBAL_EXPLICIT = 3
+SC_FREE = 4
+SC_CELL = 5
+SC_UNKNOWN = 6
+
+CO_OPTIMIZED = 0x0001
+CO_NEWLOCALS = 0x0002
+CO_VARARGS = 0x0004
+CO_VARKEYWORDS = 0x0008
+CO_NESTED = 0x0010
+CO_GENERATOR = 0x0020
+CO_GENERATOR_ALLOWED = 0
+CO_FUTURE_DIVISION = 0x2000
+CO_FUTURE_ABSIMPORT = 0x4000
+CO_FUTURE_WITH_STATEMENT = 0x8000
+CO_FUTURE_PRINT_FUNCTION = 0x10000
diff --git a/lib/python2.7/compiler/future.py b/lib/python2.7/compiler/future.py
new file mode 100644
index 0000000..fd5e5df
--- /dev/null
+++ b/lib/python2.7/compiler/future.py
@@ -0,0 +1,74 @@
+"""Parser for future statements
+
+"""
+
+from compiler import ast, walk
+
+def is_future(stmt):
+ """Return true if statement is a well-formed future statement"""
+ if not isinstance(stmt, ast.From):
+ return 0
+ if stmt.modname == "__future__":
+ return 1
+ else:
+ return 0
+
+class FutureParser:
+
+ features = ("nested_scopes", "generators", "division",
+ "absolute_import", "with_statement", "print_function",
+ "unicode_literals")
+
+ def __init__(self):
+ self.found = {} # set
+
+ def visitModule(self, node):
+ stmt = node.node
+ for s in stmt.nodes:
+ if not self.check_stmt(s):
+ break
+
+ def check_stmt(self, stmt):
+ if is_future(stmt):
+ for name, asname in stmt.names:
+ if name in self.features:
+ self.found[name] = 1
+ else:
+ raise SyntaxError, \
+ "future feature %s is not defined" % name
+ stmt.valid_future = 1
+ return 1
+ return 0
+
+ def get_features(self):
+ """Return list of features enabled by future statements"""
+ return self.found.keys()
+
+class BadFutureParser:
+ """Check for invalid future statements"""
+
+ def visitFrom(self, node):
+ if hasattr(node, 'valid_future'):
+ return
+ if node.modname != "__future__":
+ return
+ raise SyntaxError, "invalid future statement " + repr(node)
+
+def find_futures(node):
+ p1 = FutureParser()
+ p2 = BadFutureParser()
+ walk(node, p1)
+ walk(node, p2)
+ return p1.get_features()
+
+if __name__ == "__main__":
+ import sys
+ from compiler import parseFile, walk
+
+ for file in sys.argv[1:]:
+ print file
+ tree = parseFile(file)
+ v = FutureParser()
+ walk(tree, v)
+ print v.found
+ print
diff --git a/lib/python2.7/compiler/misc.py b/lib/python2.7/compiler/misc.py
new file mode 100644
index 0000000..588c7fb
--- /dev/null
+++ b/lib/python2.7/compiler/misc.py
@@ -0,0 +1,73 @@
+
+def flatten(tup):
+ elts = []
+ for elt in tup:
+ if isinstance(elt, tuple):
+ elts = elts + flatten(elt)
+ else:
+ elts.append(elt)
+ return elts
+
+class Set:
+ def __init__(self):
+ self.elts = {}
+ def __len__(self):
+ return len(self.elts)
+ def __contains__(self, elt):
+ return elt in self.elts
+ def add(self, elt):
+ self.elts[elt] = elt
+ def elements(self):
+ return self.elts.keys()
+ def has_elt(self, elt):
+ return elt in self.elts
+ def remove(self, elt):
+ del self.elts[elt]
+ def copy(self):
+ c = Set()
+ c.elts.update(self.elts)
+ return c
+
+class Stack:
+ def __init__(self):
+ self.stack = []
+ self.pop = self.stack.pop
+ def __len__(self):
+ return len(self.stack)
+ def push(self, elt):
+ self.stack.append(elt)
+ def top(self):
+ return self.stack[-1]
+ def __getitem__(self, index): # needed by visitContinue()
+ return self.stack[index]
+
+MANGLE_LEN = 256 # magic constant from compile.c
+
+def mangle(name, klass):
+ if not name.startswith('__'):
+ return name
+ if len(name) + 2 >= MANGLE_LEN:
+ return name
+ if name.endswith('__'):
+ return name
+ try:
+ i = 0
+ while klass[i] == '_':
+ i = i + 1
+ except IndexError:
+ return name
+ klass = klass[i:]
+
+ tlen = len(klass) + len(name)
+ if tlen > MANGLE_LEN:
+ klass = klass[:MANGLE_LEN-tlen]
+
+ return "_%s%s" % (klass, name)
+
+def set_filename(filename, tree):
+ """Set the filename attribute to filename on every node in tree"""
+ worklist = [tree]
+ while worklist:
+ node = worklist.pop(0)
+ node.filename = filename
+ worklist.extend(node.getChildNodes())
diff --git a/lib/python2.7/compiler/pyassem.py b/lib/python2.7/compiler/pyassem.py
new file mode 100644
index 0000000..f52f7d0
--- /dev/null
+++ b/lib/python2.7/compiler/pyassem.py
@@ -0,0 +1,763 @@
+"""A flow graph representation for Python bytecode"""
+
+import dis
+import types
+import sys
+
+from compiler import misc
+from compiler.consts \
+ import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
+
+class FlowGraph:
+ def __init__(self):
+ self.current = self.entry = Block()
+ self.exit = Block("exit")
+ self.blocks = misc.Set()
+ self.blocks.add(self.entry)
+ self.blocks.add(self.exit)
+
+ def startBlock(self, block):
+ if self._debug:
+ if self.current:
+ print "end", repr(self.current)
+ print " next", self.current.next
+ print " prev", self.current.prev
+ print " ", self.current.get_children()
+ print repr(block)
+ self.current = block
+
+ def nextBlock(self, block=None):
+ # XXX think we need to specify when there is implicit transfer
+ # from one block to the next. might be better to represent this
+ # with explicit JUMP_ABSOLUTE instructions that are optimized
+ # out when they are unnecessary.
+ #
+ # I think this strategy works: each block has a child
+ # designated as "next" which is returned as the last of the
+ # children. because the nodes in a graph are emitted in
+ # reverse post order, the "next" block will always be emitted
+ # immediately after its parent.
+ # Worry: maintaining this invariant could be tricky
+ if block is None:
+ block = self.newBlock()
+
+ # Note: If the current block ends with an unconditional control
+ # transfer, then it is techically incorrect to add an implicit
+ # transfer to the block graph. Doing so results in code generation
+ # for unreachable blocks. That doesn't appear to be very common
+ # with Python code and since the built-in compiler doesn't optimize
+ # it out we don't either.
+ self.current.addNext(block)
+ self.startBlock(block)
+
+ def newBlock(self):
+ b = Block()
+ self.blocks.add(b)
+ return b
+
+ def startExitBlock(self):
+ self.startBlock(self.exit)
+
+ _debug = 0
+
+ def _enable_debug(self):
+ self._debug = 1
+
+ def _disable_debug(self):
+ self._debug = 0
+
+ def emit(self, *inst):
+ if self._debug:
+ print "\t", inst
+ if len(inst) == 2 and isinstance(inst[1], Block):
+ self.current.addOutEdge(inst[1])
+ self.current.emit(inst)
+
+ def getBlocksInOrder(self):
+ """Return the blocks in reverse postorder
+
+ i.e. each node appears before all of its successors
+ """
+ order = order_blocks(self.entry, self.exit)
+ return order
+
+ def getBlocks(self):
+ return self.blocks.elements()
+
+ def getRoot(self):
+ """Return nodes appropriate for use with dominator"""
+ return self.entry
+
+ def getContainedGraphs(self):
+ l = []
+ for b in self.getBlocks():
+ l.extend(b.getContainedGraphs())
+ return l
+
+
+def order_blocks(start_block, exit_block):
+ """Order blocks so that they are emitted in the right order"""
+ # Rules:
+ # - when a block has a next block, the next block must be emitted just after
+ # - when a block has followers (relative jumps), it must be emitted before
+ # them
+ # - all reachable blocks must be emitted
+ order = []
+
+ # Find all the blocks to be emitted.
+ remaining = set()
+ todo = [start_block]
+ while todo:
+ b = todo.pop()
+ if b in remaining:
+ continue
+ remaining.add(b)
+ for c in b.get_children():
+ if c not in remaining:
+ todo.append(c)
+
+ # A block is dominated by another block if that block must be emitted
+ # before it.
+ dominators = {}
+ for b in remaining:
+ if __debug__ and b.next:
+ assert b is b.next[0].prev[0], (b, b.next)
+ # Make sure every block appears in dominators, even if no
+ # other block must precede it.
+ dominators.setdefault(b, set())
+ # preceding blocks dominate following blocks
+ for c in b.get_followers():
+ while 1:
+ dominators.setdefault(c, set()).add(b)
+ # Any block that has a next pointer leading to c is also
+ # dominated because the whole chain will be emitted at once.
+ # Walk backwards and add them all.
+ if c.prev and c.prev[0] is not b:
+ c = c.prev[0]
+ else:
+ break
+
+ def find_next():
+ # Find a block that can be emitted next.
+ for b in remaining:
+ for c in dominators[b]:
+ if c in remaining:
+ break # can't emit yet, dominated by a remaining block
+ else:
+ return b
+ assert 0, 'circular dependency, cannot find next block'
+
+ b = start_block
+ while 1:
+ order.append(b)
+ remaining.discard(b)
+ if b.next:
+ b = b.next[0]
+ continue
+ elif b is not exit_block and not b.has_unconditional_transfer():
+ order.append(exit_block)
+ if not remaining:
+ break
+ b = find_next()
+ return order
+
+
+class Block:
+ _count = 0
+
+ def __init__(self, label=''):
+ self.insts = []
+ self.outEdges = set()
+ self.label = label
+ self.bid = Block._count
+ self.next = []
+ self.prev = []
+ Block._count = Block._count + 1
+
+ def __repr__(self):
+ if self.label:
+ return "<block %s id=%d>" % (self.label, self.bid)
+ else:
+ return "<block id=%d>" % (self.bid)
+
+ def __str__(self):
+ insts = map(str, self.insts)
+ return "<block %s %d:\n%s>" % (self.label, self.bid,
+ '\n'.join(insts))
+
+ def emit(self, inst):
+ op = inst[0]
+ self.insts.append(inst)
+
+ def getInstructions(self):
+ return self.insts
+
+ def addOutEdge(self, block):
+ self.outEdges.add(block)
+
+ def addNext(self, block):
+ self.next.append(block)
+ assert len(self.next) == 1, map(str, self.next)
+ block.prev.append(self)
+ assert len(block.prev) == 1, map(str, block.prev)
+
+ _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
+ 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP',
+ )
+
+ def has_unconditional_transfer(self):
+ """Returns True if there is an unconditional transfer to an other block
+ at the end of this block. This means there is no risk for the bytecode
+ executer to go past this block's bytecode."""
+ try:
+ op, arg = self.insts[-1]
+ except (IndexError, ValueError):
+ return
+ return op in self._uncond_transfer
+
+ def get_children(self):
+ return list(self.outEdges) + self.next
+
+ def get_followers(self):
+ """Get the whole list of followers, including the next block."""
+ followers = set(self.next)
+ # Blocks that must be emitted *after* this one, because of
+ # bytecode offsets (e.g. relative jumps) pointing to them.
+ for inst in self.insts:
+ if inst[0] in PyFlowGraph.hasjrel:
+ followers.add(inst[1])
+ return followers
+
+ def getContainedGraphs(self):
+ """Return all graphs contained within this block.
+
+ For example, a MAKE_FUNCTION block will contain a reference to
+ the graph for the function body.
+ """
+ contained = []
+ for inst in self.insts:
+ if len(inst) == 1:
+ continue
+ op = inst[1]
+ if hasattr(op, 'graph'):
+ contained.append(op.graph)
+ return contained
+
+# flags for code objects
+
+# the FlowGraph is transformed in place; it exists in one of these states
+RAW = "RAW"
+FLAT = "FLAT"
+CONV = "CONV"
+DONE = "DONE"
+
+class PyFlowGraph(FlowGraph):
+ super_init = FlowGraph.__init__
+
+ def __init__(self, name, filename, args=(), optimized=0, klass=None):
+ self.super_init()
+ self.name = name
+ self.filename = filename
+ self.docstring = None
+ self.args = args # XXX
+ self.argcount = getArgCount(args)
+ self.klass = klass
+ if optimized:
+ self.flags = CO_OPTIMIZED | CO_NEWLOCALS
+ else:
+ self.flags = 0
+ self.consts = []
+ self.names = []
+ # Free variables found by the symbol table scan, including
+ # variables used only in nested scopes, are included here.
+ self.freevars = []
+ self.cellvars = []
+ # The closure list is used to track the order of cell
+ # variables and free variables in the resulting code object.
+ # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both
+ # kinds of variables.
+ self.closure = []
+ self.varnames = list(args) or []
+ for i in range(len(self.varnames)):
+ var = self.varnames[i]
+ if isinstance(var, TupleArg):
+ self.varnames[i] = var.getName()
+ self.stage = RAW
+
+ def setDocstring(self, doc):
+ self.docstring = doc
+
+ def setFlag(self, flag):
+ self.flags = self.flags | flag
+ if flag == CO_VARARGS:
+ self.argcount = self.argcount - 1
+
+ def checkFlag(self, flag):
+ if self.flags & flag:
+ return 1
+
+ def setFreeVars(self, names):
+ self.freevars = list(names)
+
+ def setCellVars(self, names):
+ self.cellvars = names
+
+ def getCode(self):
+ """Get a Python code object"""
+ assert self.stage == RAW
+ self.computeStackDepth()
+ self.flattenGraph()
+ assert self.stage == FLAT
+ self.convertArgs()
+ assert self.stage == CONV
+ self.makeByteCode()
+ assert self.stage == DONE
+ return self.newCodeObject()
+
+ def dump(self, io=None):
+ if io:
+ save = sys.stdout
+ sys.stdout = io
+ pc = 0
+ for t in self.insts:
+ opname = t[0]
+ if opname == "SET_LINENO":
+ print
+ if len(t) == 1:
+ print "\t", "%3d" % pc, opname
+ pc = pc + 1
+ else:
+ print "\t", "%3d" % pc, opname, t[1]
+ pc = pc + 3
+ if io:
+ sys.stdout = save
+
+ def computeStackDepth(self):
+ """Compute the max stack depth.
+
+ Approach is to compute the stack effect of each basic block.
+ Then find the path through the code with the largest total
+ effect.
+ """
+ depth = {}
+ exit = None
+ for b in self.getBlocks():
+ depth[b] = findDepth(b.getInstructions())
+
+ seen = {}
+
+ def max_depth(b, d):
+ if b in seen:
+ return d
+ seen[b] = 1
+ d = d + depth[b]
+ children = b.get_children()
+ if children:
+ return max([max_depth(c, d) for c in children])
+ else:
+ if not b.label == "exit":
+ return max_depth(self.exit, d)
+ else:
+ return d
+
+ self.stacksize = max_depth(self.entry, 0)
+
+ def flattenGraph(self):
+ """Arrange the blocks in order and resolve jumps"""
+ assert self.stage == RAW
+ self.insts = insts = []
+ pc = 0
+ begin = {}
+ end = {}
+ for b in self.getBlocksInOrder():
+ begin[b] = pc
+ for inst in b.getInstructions():
+ insts.append(inst)
+ if len(inst) == 1:
+ pc = pc + 1
+ elif inst[0] != "SET_LINENO":
+ # arg takes 2 bytes
+ pc = pc + 3
+ end[b] = pc
+ pc = 0
+ for i in range(len(insts)):
+ inst = insts[i]
+ if len(inst) == 1:
+ pc = pc + 1
+ elif inst[0] != "SET_LINENO":
+ pc = pc + 3
+ opname = inst[0]
+ if opname in self.hasjrel:
+ oparg = inst[1]
+ offset = begin[oparg] - pc
+ insts[i] = opname, offset
+ elif opname in self.hasjabs:
+ insts[i] = opname, begin[inst[1]]
+ self.stage = FLAT
+
+ hasjrel = set()
+ for i in dis.hasjrel:
+ hasjrel.add(dis.opname[i])
+ hasjabs = set()
+ for i in dis.hasjabs:
+ hasjabs.add(dis.opname[i])
+
+ def convertArgs(self):
+ """Convert arguments from symbolic to concrete form"""
+ assert self.stage == FLAT
+ self.consts.insert(0, self.docstring)
+ self.sort_cellvars()
+ for i in range(len(self.insts)):
+ t = self.insts[i]
+ if len(t) == 2:
+ opname, oparg = t
+ conv = self._converters.get(opname, None)
+ if conv:
+ self.insts[i] = opname, conv(self, oparg)
+ self.stage = CONV
+
+ def sort_cellvars(self):
+ """Sort cellvars in the order of varnames and prune from freevars.
+ """
+ cells = {}
+ for name in self.cellvars:
+ cells[name] = 1
+ self.cellvars = [name for name in self.varnames
+ if name in cells]
+ for name in self.cellvars:
+ del cells[name]
+ self.cellvars = self.cellvars + cells.keys()
+ self.closure = self.cellvars + self.freevars
+
+ def _lookupName(self, name, list):
+ """Return index of name in list, appending if necessary
+
+ This routine uses a list instead of a dictionary, because a
+ dictionary can't store two different keys if the keys have the
+ same value but different types, e.g. 2 and 2L. The compiler
+ must treat these two separately, so it does an explicit type
+ comparison before comparing the values.
+ """
+ t = type(name)
+ for i in range(len(list)):
+ if t == type(list[i]) and list[i] == name:
+ return i
+ end = len(list)
+ list.append(name)
+ return end
+
+ _converters = {}
+ def _convert_LOAD_CONST(self, arg):
+ if hasattr(arg, 'getCode'):
+ arg = arg.getCode()
+ return self._lookupName(arg, self.consts)
+
+ def _convert_LOAD_FAST(self, arg):
+ self._lookupName(arg, self.names)
+ return self._lookupName(arg, self.varnames)
+ _convert_STORE_FAST = _convert_LOAD_FAST
+ _convert_DELETE_FAST = _convert_LOAD_FAST
+
+ def _convert_LOAD_NAME(self, arg):
+ if self.klass is None:
+ self._lookupName(arg, self.varnames)
+ return self._lookupName(arg, self.names)
+
+ def _convert_NAME(self, arg):
+ if self.klass is None:
+ self._lookupName(arg, self.varnames)
+ return self._lookupName(arg, self.names)
+ _convert_STORE_NAME = _convert_NAME
+ _convert_DELETE_NAME = _convert_NAME
+ _convert_IMPORT_NAME = _convert_NAME
+ _convert_IMPORT_FROM = _convert_NAME
+ _convert_STORE_ATTR = _convert_NAME
+ _convert_LOAD_ATTR = _convert_NAME
+ _convert_DELETE_ATTR = _convert_NAME
+ _convert_LOAD_GLOBAL = _convert_NAME
+ _convert_STORE_GLOBAL = _convert_NAME
+ _convert_DELETE_GLOBAL = _convert_NAME
+
+ def _convert_DEREF(self, arg):
+ self._lookupName(arg, self.names)
+ self._lookupName(arg, self.varnames)
+ return self._lookupName(arg, self.closure)
+ _convert_LOAD_DEREF = _convert_DEREF
+ _convert_STORE_DEREF = _convert_DEREF
+
+ def _convert_LOAD_CLOSURE(self, arg):
+ self._lookupName(arg, self.varnames)
+ return self._lookupName(arg, self.closure)
+
+ _cmp = list(dis.cmp_op)
+ def _convert_COMPARE_OP(self, arg):
+ return self._cmp.index(arg)
+
+ # similarly for other opcodes...
+
+ for name, obj in locals().items():
+ if name[:9] == "_convert_":
+ opname = name[9:]
+ _converters[opname] = obj
+ del name, obj, opname
+
+ def makeByteCode(self):
+ assert self.stage == CONV
+ self.lnotab = lnotab = LineAddrTable()
+ for t in self.insts:
+ opname = t[0]
+ if len(t) == 1:
+ lnotab.addCode(self.opnum[opname])
+ else:
+ oparg = t[1]
+ if opname == "SET_LINENO":
+ lnotab.nextLine(oparg)
+ continue
+ hi, lo = twobyte(oparg)
+ try:
+ lnotab.addCode(self.opnum[opname], lo, hi)
+ except ValueError:
+ print opname, oparg
+ print self.opnum[opname], lo, hi
+ raise
+ self.stage = DONE
+
+ opnum = {}
+ for num in range(len(dis.opname)):
+ opnum[dis.opname[num]] = num
+ del num
+
+ def newCodeObject(self):
+ assert self.stage == DONE
+ if (self.flags & CO_NEWLOCALS) == 0:
+ nlocals = 0
+ else:
+ nlocals = len(self.varnames)
+ argcount = self.argcount
+ if self.flags & CO_VARKEYWORDS:
+ argcount = argcount - 1
+ return types.CodeType(argcount, nlocals, self.stacksize, self.flags,
+ self.lnotab.getCode(), self.getConsts(),
+ tuple(self.names), tuple(self.varnames),
+ self.filename, self.name, self.lnotab.firstline,
+ self.lnotab.getTable(), tuple(self.freevars),
+ tuple(self.cellvars))
+
+ def getConsts(self):
+ """Return a tuple for the const slot of the code object
+
+ Must convert references to code (MAKE_FUNCTION) to code
+ objects recursively.
+ """
+ l = []
+ for elt in self.consts:
+ if isinstance(elt, PyFlowGraph):
+ elt = elt.getCode()
+ l.append(elt)
+ return tuple(l)
+
+def isJump(opname):
+ if opname[:4] == 'JUMP':
+ return 1
+
+class TupleArg:
+ """Helper for marking func defs with nested tuples in arglist"""
+ def __init__(self, count, names):
+ self.count = count
+ self.names = names
+ def __repr__(self):
+ return "TupleArg(%s, %s)" % (self.count, self.names)
+ def getName(self):
+ return ".%d" % self.count
+
+def getArgCount(args):
+ argcount = len(args)
+ if args:
+ for arg in args:
+ if isinstance(arg, TupleArg):
+ numNames = len(misc.flatten(arg.names))
+ argcount = argcount - numNames
+ return argcount
+
+def twobyte(val):
+ """Convert an int argument into high and low bytes"""
+ assert isinstance(val, int)
+ return divmod(val, 256)
+
+class LineAddrTable:
+ """lnotab
+
+ This class builds the lnotab, which is documented in compile.c.
+ Here's a brief recap:
+
+ For each SET_LINENO instruction after the first one, two bytes are
+ added to lnotab. (In some cases, multiple two-byte entries are
+ added.) The first byte is the distance in bytes between the
+ instruction for the last SET_LINENO and the current SET_LINENO.
+ The second byte is offset in line numbers. If either offset is
+ greater than 255, multiple two-byte entries are added -- see
+ compile.c for the delicate details.
+ """
+
+ def __init__(self):
+ self.code = []
+ self.codeOffset = 0
+ self.firstline = 0
+ self.lastline = 0
+ self.lastoff = 0
+ self.lnotab = []
+
+ def addCode(self, *args):
+ for arg in args:
+ self.code.append(chr(arg))
+ self.codeOffset = self.codeOffset + len(args)
+
+ def nextLine(self, lineno):
+ if self.firstline == 0:
+ self.firstline = lineno
+ self.lastline = lineno
+ else:
+ # compute deltas
+ addr = self.codeOffset - self.lastoff
+ line = lineno - self.lastline
+ # Python assumes that lineno always increases with
+ # increasing bytecode address (lnotab is unsigned char).
+ # Depending on when SET_LINENO instructions are emitted
+ # this is not always true. Consider the code:
+ # a = (1,
+ # b)
+ # In the bytecode stream, the assignment to "a" occurs
+ # after the loading of "b". This works with the C Python
+ # compiler because it only generates a SET_LINENO instruction
+ # for the assignment.
+ if line >= 0:
+ push = self.lnotab.append
+ while addr > 255:
+ push(255); push(0)
+ addr -= 255
+ while line > 255:
+ push(addr); push(255)
+ line -= 255
+ addr = 0
+ if addr > 0 or line > 0:
+ push(addr); push(line)
+ self.lastline = lineno
+ self.lastoff = self.codeOffset
+
+ def getCode(self):
+ return ''.join(self.code)
+
+ def getTable(self):
+ return ''.join(map(chr, self.lnotab))
+
+class StackDepthTracker:
+ # XXX 1. need to keep track of stack depth on jumps
+ # XXX 2. at least partly as a result, this code is broken
+
+ def findDepth(self, insts, debug=0):
+ depth = 0
+ maxDepth = 0
+ for i in insts:
+ opname = i[0]
+ if debug:
+ print i,
+ delta = self.effect.get(opname, None)
+ if delta is not None:
+ depth = depth + delta
+ else:
+ # now check patterns
+ for pat, pat_delta in self.patterns:
+ if opname[:len(pat)] == pat:
+ delta = pat_delta
+ depth = depth + delta
+ break
+ # if we still haven't found a match
+ if delta is None:
+ meth = getattr(self, opname, None)
+ if meth is not None:
+ depth = depth + meth(i[1])
+ if depth > maxDepth:
+ maxDepth = depth
+ if debug:
+ print depth, maxDepth
+ return maxDepth
+
+ effect = {
+ 'POP_TOP': -1,
+ 'DUP_TOP': 1,
+ 'LIST_APPEND': -1,
+ 'SET_ADD': -1,
+ 'MAP_ADD': -2,
+ 'SLICE+1': -1,
+ 'SLICE+2': -1,
+ 'SLICE+3': -2,
+ 'STORE_SLICE+0': -1,
+ 'STORE_SLICE+1': -2,
+ 'STORE_SLICE+2': -2,
+ 'STORE_SLICE+3': -3,
+ 'DELETE_SLICE+0': -1,
+ 'DELETE_SLICE+1': -2,
+ 'DELETE_SLICE+2': -2,
+ 'DELETE_SLICE+3': -3,
+ 'STORE_SUBSCR': -3,
+ 'DELETE_SUBSCR': -2,
+ # PRINT_EXPR?
+ 'PRINT_ITEM': -1,
+ 'RETURN_VALUE': -1,
+ 'YIELD_VALUE': -1,
+ 'EXEC_STMT': -3,
+ 'BUILD_CLASS': -2,
+ 'STORE_NAME': -1,
+ 'STORE_ATTR': -2,
+ 'DELETE_ATTR': -1,
+ 'STORE_GLOBAL': -1,
+ 'BUILD_MAP': 1,
+ 'COMPARE_OP': -1,
+ 'STORE_FAST': -1,
+ 'IMPORT_STAR': -1,
+ 'IMPORT_NAME': -1,
+ 'IMPORT_FROM': 1,
+ 'LOAD_ATTR': 0, # unlike other loads
+ # close enough...
+ 'SETUP_EXCEPT': 3,
+ 'SETUP_FINALLY': 3,
+ 'FOR_ITER': 1,
+ 'WITH_CLEANUP': -1,
+ }
+ # use pattern match
+ patterns = [
+ ('BINARY_', -1),
+ ('LOAD_', 1),
+ ]
+
+ def UNPACK_SEQUENCE(self, count):
+ return count-1
+ def BUILD_TUPLE(self, count):
+ return -count+1
+ def BUILD_LIST(self, count):
+ return -count+1
+ def BUILD_SET(self, count):
+ return -count+1
+ def CALL_FUNCTION(self, argc):
+ hi, lo = divmod(argc, 256)
+ return -(lo + hi * 2)
+ def CALL_FUNCTION_VAR(self, argc):
+ return self.CALL_FUNCTION(argc)-1
+ def CALL_FUNCTION_KW(self, argc):
+ return self.CALL_FUNCTION(argc)-1
+ def CALL_FUNCTION_VAR_KW(self, argc):
+ return self.CALL_FUNCTION(argc)-2
+ def MAKE_FUNCTION(self, argc):
+ return -argc
+ def MAKE_CLOSURE(self, argc):
+ # XXX need to account for free variables too!
+ return -argc
+ def BUILD_SLICE(self, argc):
+ if argc == 2:
+ return -1
+ elif argc == 3:
+ return -2
+ def DUP_TOPX(self, argc):
+ return argc
+
+findDepth = StackDepthTracker().findDepth
diff --git a/lib/python2.7/compiler/pycodegen.py b/lib/python2.7/compiler/pycodegen.py
new file mode 100644
index 0000000..6515945
--- /dev/null
+++ b/lib/python2.7/compiler/pycodegen.py
@@ -0,0 +1,1555 @@
+import imp
+import os
+import marshal
+import struct
+import sys
+from cStringIO import StringIO
+
+from compiler import ast, parse, walk, syntax
+from compiler import pyassem, misc, future, symbols
+from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \
+ SC_FREE, SC_CELL
+from compiler.consts import (CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS,
+ CO_NESTED, CO_GENERATOR, CO_FUTURE_DIVISION,
+ CO_FUTURE_ABSIMPORT, CO_FUTURE_WITH_STATEMENT, CO_FUTURE_PRINT_FUNCTION)
+from compiler.pyassem import TupleArg
+
+# XXX The version-specific code can go, since this code only works with 2.x.
+# Do we have Python 1.x or Python 2.x?
+try:
+ VERSION = sys.version_info[0]
+except AttributeError:
+ VERSION = 1
+
+callfunc_opcode_info = {
+ # (Have *args, Have **args) : opcode
+ (0,0) : "CALL_FUNCTION",
+ (1,0) : "CALL_FUNCTION_VAR",
+ (0,1) : "CALL_FUNCTION_KW",
+ (1,1) : "CALL_FUNCTION_VAR_KW",
+}
+
+LOOP = 1
+EXCEPT = 2
+TRY_FINALLY = 3
+END_FINALLY = 4
+
+def compileFile(filename, display=0):
+ f = open(filename, 'U')
+ buf = f.read()
+ f.close()
+ mod = Module(buf, filename)
+ try:
+ mod.compile(display)
+ except SyntaxError:
+ raise
+ else:
+ f = open(filename + "c", "wb")
+ mod.dump(f)
+ f.close()
+
+def compile(source, filename, mode, flags=None, dont_inherit=None):
+ """Replacement for builtin compile() function"""
+ if flags is not None or dont_inherit is not None:
+ raise RuntimeError, "not implemented yet"
+
+ if mode == "single":
+ gen = Interactive(source, filename)
+ elif mode == "exec":
+ gen = Module(source, filename)
+ elif mode == "eval":
+ gen = Expression(source, filename)
+ else:
+ raise ValueError("compile() 3rd arg must be 'exec' or "
+ "'eval' or 'single'")
+ gen.compile()
+ return gen.code
+
+class AbstractCompileMode:
+
+ mode = None # defined by subclass
+
+ def __init__(self, source, filename):
+ self.source = source
+ self.filename = filename
+ self.code = None
+
+ def _get_tree(self):
+ tree = parse(self.source, self.mode)
+ misc.set_filename(self.filename, tree)
+ syntax.check(tree)
+ return tree
+
+ def compile(self):
+ pass # implemented by subclass
+
+ def getCode(self):
+ return self.code
+
+class Expression(AbstractCompileMode):
+
+ mode = "eval"
+
+ def compile(self):
+ tree = self._get_tree()
+ gen = ExpressionCodeGenerator(tree)
+ self.code = gen.getCode()
+
+class Interactive(AbstractCompileMode):
+
+ mode = "single"
+
+ def compile(self):
+ tree = self._get_tree()
+ gen = InteractiveCodeGenerator(tree)
+ self.code = gen.getCode()
+
+class Module(AbstractCompileMode):
+
+ mode = "exec"
+
+ def compile(self, display=0):
+ tree = self._get_tree()
+ gen = ModuleCodeGenerator(tree)
+ if display:
+ import pprint
+ print pprint.pprint(tree)
+ self.code = gen.getCode()
+
+ def dump(self, f):
+ f.write(self.getPycHeader())
+ marshal.dump(self.code, f)
+
+ MAGIC = imp.get_magic()
+
+ def getPycHeader(self):
+ # compile.c uses marshal to write a long directly, with
+ # calling the interface that would also generate a 1-byte code
+ # to indicate the type of the value. simplest way to get the
+ # same effect is to call marshal and then skip the code.
+ mtime = os.path.getmtime(self.filename)
+ mtime = struct.pack('<i', mtime)
+ return self.MAGIC + mtime
+
+class LocalNameFinder:
+ """Find local names in scope"""
+ def __init__(self, names=()):
+ self.names = misc.Set()
+ self.globals = misc.Set()
+ for name in names:
+ self.names.add(name)
+
+ # XXX list comprehensions and for loops
+
+ def getLocals(self):
+ for elt in self.globals.elements():
+ if self.names.has_elt(elt):
+ self.names.remove(elt)
+ return self.names
+
+ def visitDict(self, node):
+ pass
+
+ def visitGlobal(self, node):
+ for name in node.names:
+ self.globals.add(name)
+
+ def visitFunction(self, node):
+ self.names.add(node.name)
+
+ def visitLambda(self, node):
+ pass
+
+ def visitImport(self, node):
+ for name, alias in node.names:
+ self.names.add(alias or name)
+
+ def visitFrom(self, node):
+ for name, alias in node.names:
+ self.names.add(alias or name)
+
+ def visitClass(self, node):
+ self.names.add(node.name)
+
+ def visitAssName(self, node):
+ self.names.add(node.name)
+
+def is_constant_false(node):
+ if isinstance(node, ast.Const):
+ if not node.value:
+ return 1
+ return 0
+
+class CodeGenerator:
+ """Defines basic code generator for Python bytecode
+
+ This class is an abstract base class. Concrete subclasses must
+ define an __init__() that defines self.graph and then calls the
+ __init__() defined in this class.
+
+ The concrete class must also define the class attributes
+ NameFinder, FunctionGen, and ClassGen. These attributes can be
+ defined in the initClass() method, which is a hook for
+ initializing these methods after all the classes have been
+ defined.
+ """
+
+ optimized = 0 # is namespace access optimized?
+ __initialized = None
+ class_name = None # provide default for instance variable
+
+ def __init__(self):
+ if self.__initialized is None:
+ self.initClass()
+ self.__class__.__initialized = 1
+ self.checkClass()
+ self.locals = misc.Stack()
+ self.setups = misc.Stack()
+ self.last_lineno = None
+ self._setupGraphDelegation()
+ self._div_op = "BINARY_DIVIDE"
+
+ # XXX set flags based on future features
+ futures = self.get_module().futures
+ for feature in futures:
+ if feature == "division":
+ self.graph.setFlag(CO_FUTURE_DIVISION)
+ self._div_op = "BINARY_TRUE_DIVIDE"
+ elif feature == "absolute_import":
+ self.graph.setFlag(CO_FUTURE_ABSIMPORT)
+ elif feature == "with_statement":
+ self.graph.setFlag(CO_FUTURE_WITH_STATEMENT)
+ elif feature == "print_function":
+ self.graph.setFlag(CO_FUTURE_PRINT_FUNCTION)
+
+ def initClass(self):
+ """This method is called once for each class"""
+
+ def checkClass(self):
+ """Verify that class is constructed correctly"""
+ try:
+ assert hasattr(self, 'graph')
+ assert getattr(self, 'NameFinder')
+ assert getattr(self, 'FunctionGen')
+ assert getattr(self, 'ClassGen')
+ except AssertionError, msg:
+ intro = "Bad class construction for %s" % self.__class__.__name__
+ raise AssertionError, intro
+
+ def _setupGraphDelegation(self):
+ self.emit = self.graph.emit
+ self.newBlock = self.graph.newBlock
+ self.startBlock = self.graph.startBlock
+ self.nextBlock = self.graph.nextBlock
+ self.setDocstring = self.graph.setDocstring
+
+ def getCode(self):
+ """Return a code object"""
+ return self.graph.getCode()
+
+ def mangle(self, name):
+ if self.class_name is not None:
+ return misc.mangle(name, self.class_name)
+ else:
+ return name
+
+ def parseSymbols(self, tree):
+ s = symbols.SymbolVisitor()
+ walk(tree, s)
+ return s.scopes
+
+ def get_module(self):
+ raise RuntimeError, "should be implemented by subclasses"
+
+ # Next five methods handle name access
+
+ def isLocalName(self, name):
+ return self.locals.top().has_elt(name)
+
+ def storeName(self, name):
+ self._nameOp('STORE', name)
+
+ def loadName(self, name):
+ self._nameOp('LOAD', name)
+
+ def delName(self, name):
+ self._nameOp('DELETE', name)
+
+ def _nameOp(self, prefix, name):
+ name = self.mangle(name)
+ scope = self.scope.check_name(name)
+ if scope == SC_LOCAL:
+ if not self.optimized:
+ self.emit(prefix + '_NAME', name)
+ else:
+ self.emit(prefix + '_FAST', name)
+ elif scope == SC_GLOBAL_EXPLICIT:
+ self.emit(prefix + '_GLOBAL', name)
+ elif scope == SC_GLOBAL_IMPLICIT:
+ if not self.optimized:
+ self.emit(prefix + '_NAME', name)
+ else:
+ self.emit(prefix + '_GLOBAL', name)
+ elif scope == SC_FREE or scope == SC_CELL:
+ self.emit(prefix + '_DEREF', name)
+ else:
+ raise RuntimeError, "unsupported scope for var %s: %d" % \
+ (name, scope)
+
+ def _implicitNameOp(self, prefix, name):
+ """Emit name ops for names generated implicitly by for loops
+
+ The interpreter generates names that start with a period or
+ dollar sign. The symbol table ignores these names because
+ they aren't present in the program text.
+ """
+ if self.optimized:
+ self.emit(prefix + '_FAST', name)
+ else:
+ self.emit(prefix + '_NAME', name)
+
+ # The set_lineno() function and the explicit emit() calls for
+ # SET_LINENO below are only used to generate the line number table.
+ # As of Python 2.3, the interpreter does not have a SET_LINENO
+ # instruction. pyassem treats SET_LINENO opcodes as a special case.
+
+ def set_lineno(self, node, force=False):
+ """Emit SET_LINENO if necessary.
+
+ The instruction is considered necessary if the node has a
+ lineno attribute and it is different than the last lineno
+ emitted.
+
+ Returns true if SET_LINENO was emitted.
+
+ There are no rules for when an AST node should have a lineno
+ attribute. The transformer and AST code need to be reviewed
+ and a consistent policy implemented and documented. Until
+ then, this method works around missing line numbers.
+ """
+ lineno = getattr(node, 'lineno', None)
+ if lineno is not None and (lineno != self.last_lineno
+ or force):
+ self.emit('SET_LINENO', lineno)
+ self.last_lineno = lineno
+ return True
+ return False
+
+ # The first few visitor methods handle nodes that generator new
+ # code objects. They use class attributes to determine what
+ # specialized code generators to use.
+
+ NameFinder = LocalNameFinder
+ FunctionGen = None
+ ClassGen = None
+
+ def visitModule(self, node):
+ self.scopes = self.parseSymbols(node)
+ self.scope = self.scopes[node]
+ self.emit('SET_LINENO', 0)
+ if node.doc:
+ self.emit('LOAD_CONST', node.doc)
+ self.storeName('__doc__')
+ lnf = walk(node.node, self.NameFinder(), verbose=0)
+ self.locals.push(lnf.getLocals())
+ self.visit(node.node)
+ self.emit('LOAD_CONST', None)
+ self.emit('RETURN_VALUE')
+
+ def visitExpression(self, node):
+ self.set_lineno(node)
+ self.scopes = self.parseSymbols(node)
+ self.scope = self.scopes[node]
+ self.visit(node.node)
+ self.emit('RETURN_VALUE')
+
+ def visitFunction(self, node):
+ self._visitFuncOrLambda(node, isLambda=0)
+ if node.doc:
+ self.setDocstring(node.doc)
+ self.storeName(node.name)
+
+ def visitLambda(self, node):
+ self._visitFuncOrLambda(node, isLambda=1)
+
+ def _visitFuncOrLambda(self, node, isLambda=0):
+ if not isLambda and node.decorators:
+ for decorator in node.decorators.nodes:
+ self.visit(decorator)
+ ndecorators = len(node.decorators.nodes)
+ else:
+ ndecorators = 0
+
+ gen = self.FunctionGen(node, self.scopes, isLambda,
+ self.class_name, self.get_module())
+ walk(node.code, gen)
+ gen.finish()
+ self.set_lineno(node)
+ for default in node.defaults:
+ self.visit(default)
+ self._makeClosure(gen, len(node.defaults))
+ for i in range(ndecorators):
+ self.emit('CALL_FUNCTION', 1)
+
+ def visitClass(self, node):
+ gen = self.ClassGen(node, self.scopes,
+ self.get_module())
+ walk(node.code, gen)
+ gen.finish()
+ self.set_lineno(node)
+ self.emit('LOAD_CONST', node.name)
+ for base in node.bases:
+ self.visit(base)
+ self.emit('BUILD_TUPLE', len(node.bases))
+ self._makeClosure(gen, 0)
+ self.emit('CALL_FUNCTION', 0)
+ self.emit('BUILD_CLASS')
+ self.storeName(node.name)
+
+ # The rest are standard visitor methods
+
+ # The next few implement control-flow statements
+
+ def visitIf(self, node):
+ end = self.newBlock()
+ numtests = len(node.tests)
+ for i in range(numtests):
+ test, suite = node.tests[i]
+ if is_constant_false(test):
+ # XXX will need to check generator stuff here
+ continue
+ self.set_lineno(test)
+ self.visit(test)
+ nextTest = self.newBlock()
+ self.emit('POP_JUMP_IF_FALSE', nextTest)
+ self.nextBlock()
+ self.visit(suite)
+ self.emit('JUMP_FORWARD', end)
+ self.startBlock(nextTest)
+ if node.else_:
+ self.visit(node.else_)
+ self.nextBlock(end)
+
+ def visitWhile(self, node):
+ self.set_lineno(node)
+
+ loop = self.newBlock()
+ else_ = self.newBlock()
+
+ after = self.newBlock()
+ self.emit('SETUP_LOOP', after)
+
+ self.nextBlock(loop)
+ self.setups.push((LOOP, loop))
+
+ self.set_lineno(node, force=True)
+ self.visit(node.test)
+ self.emit('POP_JUMP_IF_FALSE', else_ or after)
+
+ self.nextBlock()
+ self.visit(node.body)
+ self.emit('JUMP_ABSOLUTE', loop)
+
+ self.startBlock(else_) # or just the POPs if not else clause
+ self.emit('POP_BLOCK')
+ self.setups.pop()
+ if node.else_:
+ self.visit(node.else_)
+ self.nextBlock(after)
+
+ def visitFor(self, node):
+ start = self.newBlock()
+ anchor = self.newBlock()
+ after = self.newBlock()
+ self.setups.push((LOOP, start))
+
+ self.set_lineno(node)
+ self.emit('SETUP_LOOP', after)
+ self.visit(node.list)
+ self.emit('GET_ITER')
+
+ self.nextBlock(start)
+ self.set_lineno(node, force=1)
+ self.emit('FOR_ITER', anchor)
+ self.visit(node.assign)
+ self.visit(node.body)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.nextBlock(anchor)
+ self.emit('POP_BLOCK')
+ self.setups.pop()
+ if node.else_:
+ self.visit(node.else_)
+ self.nextBlock(after)
+
+ def visitBreak(self, node):
+ if not self.setups:
+ raise SyntaxError, "'break' outside loop (%s, %d)" % \
+ (node.filename, node.lineno)
+ self.set_lineno(node)
+ self.emit('BREAK_LOOP')
+
+ def visitContinue(self, node):
+ if not self.setups:
+ raise SyntaxError, "'continue' outside loop (%s, %d)" % \
+ (node.filename, node.lineno)
+ kind, block = self.setups.top()
+ if kind == LOOP:
+ self.set_lineno(node)
+ self.emit('JUMP_ABSOLUTE', block)
+ self.nextBlock()
+ elif kind == EXCEPT or kind == TRY_FINALLY:
+ self.set_lineno(node)
+ # find the block that starts the loop
+ top = len(self.setups)
+ while top > 0:
+ top = top - 1
+ kind, loop_block = self.setups[top]
+ if kind == LOOP:
+ break
+ if kind != LOOP:
+ raise SyntaxError, "'continue' outside loop (%s, %d)" % \
+ (node.filename, node.lineno)
+ self.emit('CONTINUE_LOOP', loop_block)
+ self.nextBlock()
+ elif kind == END_FINALLY:
+ msg = "'continue' not allowed inside 'finally' clause (%s, %d)"
+ raise SyntaxError, msg % (node.filename, node.lineno)
+
+ def visitTest(self, node, jump):
+ end = self.newBlock()
+ for child in node.nodes[:-1]:
+ self.visit(child)
+ self.emit(jump, end)
+ self.nextBlock()
+ self.visit(node.nodes[-1])
+ self.nextBlock(end)
+
+ def visitAnd(self, node):
+ self.visitTest(node, 'JUMP_IF_FALSE_OR_POP')
+
+ def visitOr(self, node):
+ self.visitTest(node, 'JUMP_IF_TRUE_OR_POP')
+
+ def visitIfExp(self, node):
+ endblock = self.newBlock()
+ elseblock = self.newBlock()
+ self.visit(node.test)
+ self.emit('POP_JUMP_IF_FALSE', elseblock)
+ self.visit(node.then)
+ self.emit('JUMP_FORWARD', endblock)
+ self.nextBlock(elseblock)
+ self.visit(node.else_)
+ self.nextBlock(endblock)
+
+ def visitCompare(self, node):
+ self.visit(node.expr)
+ cleanup = self.newBlock()
+ for op, code in node.ops[:-1]:
+ self.visit(code)
+ self.emit('DUP_TOP')
+ self.emit('ROT_THREE')
+ self.emit('COMPARE_OP', op)
+ self.emit('JUMP_IF_FALSE_OR_POP', cleanup)
+ self.nextBlock()
+ # now do the last comparison
+ if node.ops:
+ op, code = node.ops[-1]
+ self.visit(code)
+ self.emit('COMPARE_OP', op)
+ if len(node.ops) > 1:
+ end = self.newBlock()
+ self.emit('JUMP_FORWARD', end)
+ self.startBlock(cleanup)
+ self.emit('ROT_TWO')
+ self.emit('POP_TOP')
+ self.nextBlock(end)
+
+ # list comprehensions
+ def visitListComp(self, node):
+ self.set_lineno(node)
+ # setup list
+ self.emit('BUILD_LIST', 0)
+
+ stack = []
+ for i, for_ in zip(range(len(node.quals)), node.quals):
+ start, anchor = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.visit(node.expr)
+ self.emit('LIST_APPEND', len(node.quals) + 1)
+
+ for start, cont, anchor in stack:
+ if cont:
+ self.nextBlock(cont)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.startBlock(anchor)
+
+ def visitSetComp(self, node):
+ self.set_lineno(node)
+ # setup list
+ self.emit('BUILD_SET', 0)
+
+ stack = []
+ for i, for_ in zip(range(len(node.quals)), node.quals):
+ start, anchor = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.visit(node.expr)
+ self.emit('SET_ADD', len(node.quals) + 1)
+
+ for start, cont, anchor in stack:
+ if cont:
+ self.nextBlock(cont)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.startBlock(anchor)
+
+ def visitDictComp(self, node):
+ self.set_lineno(node)
+ # setup list
+ self.emit('BUILD_MAP', 0)
+
+ stack = []
+ for i, for_ in zip(range(len(node.quals)), node.quals):
+ start, anchor = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.visit(node.value)
+ self.visit(node.key)
+ self.emit('MAP_ADD', len(node.quals) + 1)
+
+ for start, cont, anchor in stack:
+ if cont:
+ self.nextBlock(cont)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.startBlock(anchor)
+
+ def visitListCompFor(self, node):
+ start = self.newBlock()
+ anchor = self.newBlock()
+
+ self.visit(node.list)
+ self.emit('GET_ITER')
+ self.nextBlock(start)
+ self.set_lineno(node, force=True)
+ self.emit('FOR_ITER', anchor)
+ self.nextBlock()
+ self.visit(node.assign)
+ return start, anchor
+
+ def visitListCompIf(self, node, branch):
+ self.set_lineno(node, force=True)
+ self.visit(node.test)
+ self.emit('POP_JUMP_IF_FALSE', branch)
+ self.newBlock()
+
+ def _makeClosure(self, gen, args):
+ frees = gen.scope.get_free_vars()
+ if frees:
+ for name in frees:
+ self.emit('LOAD_CLOSURE', name)
+ self.emit('BUILD_TUPLE', len(frees))
+ self.emit('LOAD_CONST', gen)
+ self.emit('MAKE_CLOSURE', args)
+ else:
+ self.emit('LOAD_CONST', gen)
+ self.emit('MAKE_FUNCTION', args)
+
+ def visitGenExpr(self, node):
+ gen = GenExprCodeGenerator(node, self.scopes, self.class_name,
+ self.get_module())
+ walk(node.code, gen)
+ gen.finish()
+ self.set_lineno(node)
+ self._makeClosure(gen, 0)
+ # precomputation of outmost iterable
+ self.visit(node.code.quals[0].iter)
+ self.emit('GET_ITER')
+ self.emit('CALL_FUNCTION', 1)
+
+ def visitGenExprInner(self, node):
+ self.set_lineno(node)
+ # setup list
+
+ stack = []
+ for i, for_ in zip(range(len(node.quals)), node.quals):
+ start, anchor, end = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor, end))
+
+ self.visit(node.expr)
+ self.emit('YIELD_VALUE')
+ self.emit('POP_TOP')
+
+ for start, cont, anchor, end in stack:
+ if cont:
+ self.nextBlock(cont)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.startBlock(anchor)
+ self.emit('POP_BLOCK')
+ self.setups.pop()
+ self.nextBlock(end)
+
+ self.emit('LOAD_CONST', None)
+
+ def visitGenExprFor(self, node):
+ start = self.newBlock()
+ anchor = self.newBlock()
+ end = self.newBlock()
+
+ self.setups.push((LOOP, start))
+ self.emit('SETUP_LOOP', end)
+
+ if node.is_outmost:
+ self.loadName('.0')
+ else:
+ self.visit(node.iter)
+ self.emit('GET_ITER')
+
+ self.nextBlock(start)
+ self.set_lineno(node, force=True)
+ self.emit('FOR_ITER', anchor)
+ self.nextBlock()
+ self.visit(node.assign)
+ return start, anchor, end
+
+ def visitGenExprIf(self, node, branch):
+ self.set_lineno(node, force=True)
+ self.visit(node.test)
+ self.emit('POP_JUMP_IF_FALSE', branch)
+ self.newBlock()
+
+ # exception related
+
+ def visitAssert(self, node):
+ # XXX would be interesting to implement this via a
+ # transformation of the AST before this stage
+ if __debug__:
+ end = self.newBlock()
+ self.set_lineno(node)
+ # XXX AssertionError appears to be special case -- it is always
+ # loaded as a global even if there is a local name. I guess this
+ # is a sort of renaming op.
+ self.nextBlock()
+ self.visit(node.test)
+ self.emit('POP_JUMP_IF_TRUE', end)
+ self.nextBlock()
+ self.emit('LOAD_GLOBAL', 'AssertionError')
+ if node.fail:
+ self.visit(node.fail)
+ self.emit('RAISE_VARARGS', 2)
+ else:
+ self.emit('RAISE_VARARGS', 1)
+ self.nextBlock(end)
+
+ def visitRaise(self, node):
+ self.set_lineno(node)
+ n = 0
+ if node.expr1:
+ self.visit(node.expr1)
+ n = n + 1
+ if node.expr2:
+ self.visit(node.expr2)
+ n = n + 1
+ if node.expr3:
+ self.visit(node.expr3)
+ n = n + 1
+ self.emit('RAISE_VARARGS', n)
+
+ def visitTryExcept(self, node):
+ body = self.newBlock()
+ handlers = self.newBlock()
+ end = self.newBlock()
+ if node.else_:
+ lElse = self.newBlock()
+ else:
+ lElse = end
+ self.set_lineno(node)
+ self.emit('SETUP_EXCEPT', handlers)
+ self.nextBlock(body)
+ self.setups.push((EXCEPT, body))
+ self.visit(node.body)
+ self.emit('POP_BLOCK')
+ self.setups.pop()
+ self.emit('JUMP_FORWARD', lElse)
+ self.startBlock(handlers)
+
+ last = len(node.handlers) - 1
+ for i in range(len(node.handlers)):
+ expr, target, body = node.handlers[i]
+ self.set_lineno(expr)
+ if expr:
+ self.emit('DUP_TOP')
+ self.visit(expr)
+ self.emit('COMPARE_OP', 'exception match')
+ next = self.newBlock()
+ self.emit('POP_JUMP_IF_FALSE', next)
+ self.nextBlock()
+ self.emit('POP_TOP')
+ if target:
+ self.visit(target)
+ else:
+ self.emit('POP_TOP')
+ self.emit('POP_TOP')
+ self.visit(body)
+ self.emit('JUMP_FORWARD', end)
+ if expr:
+ self.nextBlock(next)
+ else:
+ self.nextBlock()
+ self.emit('END_FINALLY')
+ if node.else_:
+ self.nextBlock(lElse)
+ self.visit(node.else_)
+ self.nextBlock(end)
+
+ def visitTryFinally(self, node):
+ body = self.newBlock()
+ final = self.newBlock()
+ self.set_lineno(node)
+ self.emit('SETUP_FINALLY', final)
+ self.nextBlock(body)
+ self.setups.push((TRY_FINALLY, body))
+ self.visit(node.body)
+ self.emit('POP_BLOCK')
+ self.setups.pop()
+ self.emit('LOAD_CONST', None)
+ self.nextBlock(final)
+ self.setups.push((END_FINALLY, final))
+ self.visit(node.final)
+ self.emit('END_FINALLY')
+ self.setups.pop()
+
+ __with_count = 0
+
+ def visitWith(self, node):
+ body = self.newBlock()
+ final = self.newBlock()
+ self.__with_count += 1
+ valuevar = "_[%d]" % self.__with_count
+ self.set_lineno(node)
+ self.visit(node.expr)
+ self.emit('DUP_TOP')
+ self.emit('LOAD_ATTR', '__exit__')
+ self.emit('ROT_TWO')
+ self.emit('LOAD_ATTR', '__enter__')
+ self.emit('CALL_FUNCTION', 0)
+ if node.vars is None:
+ self.emit('POP_TOP')
+ else:
+ self._implicitNameOp('STORE', valuevar)
+ self.emit('SETUP_FINALLY', final)
+ self.nextBlock(body)
+ self.setups.push((TRY_FINALLY, body))
+ if node.vars is not None:
+ self._implicitNameOp('LOAD', valuevar)
+ self._implicitNameOp('DELETE', valuevar)
+ self.visit(node.vars)
+ self.visit(node.body)
+ self.emit('POP_BLOCK')
+ self.setups.pop()
+ self.emit('LOAD_CONST', None)
+ self.nextBlock(final)
+ self.setups.push((END_FINALLY, final))
+ self.emit('WITH_CLEANUP')
+ self.emit('END_FINALLY')
+ self.setups.pop()
+ self.__with_count -= 1
+
+ # misc
+
+ def visitDiscard(self, node):
+ self.set_lineno(node)
+ self.visit(node.expr)
+ self.emit('POP_TOP')
+
+ def visitConst(self, node):
+ self.emit('LOAD_CONST', node.value)
+
+ def visitKeyword(self, node):
+ self.emit('LOAD_CONST', node.name)
+ self.visit(node.expr)
+
+ def visitGlobal(self, node):
+ # no code to generate
+ pass
+
+ def visitName(self, node):
+ self.set_lineno(node)
+ self.loadName(node.name)
+
+ def visitPass(self, node):
+ self.set_lineno(node)
+
+ def visitImport(self, node):
+ self.set_lineno(node)
+ level = 0 if self.graph.checkFlag(CO_FUTURE_ABSIMPORT) else -1
+ for name, alias in node.names:
+ if VERSION > 1:
+ self.emit('LOAD_CONST', level)
+ self.emit('LOAD_CONST', None)
+ self.emit('IMPORT_NAME', name)
+ mod = name.split(".")[0]
+ if alias:
+ self._resolveDots(name)
+ self.storeName(alias)
+ else:
+ self.storeName(mod)
+
+ def visitFrom(self, node):
+ self.set_lineno(node)
+ level = node.level
+ if level == 0 and not self.graph.checkFlag(CO_FUTURE_ABSIMPORT):
+ level = -1
+ fromlist = tuple(name for (name, alias) in node.names)
+ if VERSION > 1:
+ self.emit('LOAD_CONST', level)
+ self.emit('LOAD_CONST', fromlist)
+ self.emit('IMPORT_NAME', node.modname)
+ for name, alias in node.names:
+ if VERSION > 1:
+ if name == '*':
+ self.namespace = 0
+ self.emit('IMPORT_STAR')
+ # There can only be one name w/ from ... import *
+ assert len(node.names) == 1
+ return
+ else:
+ self.emit('IMPORT_FROM', name)
+ self._resolveDots(name)
+ self.storeName(alias or name)
+ else:
+ self.emit('IMPORT_FROM', name)
+ self.emit('POP_TOP')
+
+ def _resolveDots(self, name):
+ elts = name.split(".")
+ if len(elts) == 1:
+ return
+ for elt in elts[1:]:
+ self.emit('LOAD_ATTR', elt)
+
+ def visitGetattr(self, node):
+ self.visit(node.expr)
+ self.emit('LOAD_ATTR', self.mangle(node.attrname))
+
+ # next five implement assignments
+
+ def visitAssign(self, node):
+ self.set_lineno(node)
+ self.visit(node.expr)
+ dups = len(node.nodes) - 1
+ for i in range(len(node.nodes)):
+ elt = node.nodes[i]
+ if i < dups:
+ self.emit('DUP_TOP')
+ if isinstance(elt, ast.Node):
+ self.visit(elt)
+
+ def visitAssName(self, node):
+ if node.flags == 'OP_ASSIGN':
+ self.storeName(node.name)
+ elif node.flags == 'OP_DELETE':
+ self.set_lineno(node)
+ self.delName(node.name)
+ else:
+ print "oops", node.flags
+
+ def visitAssAttr(self, node):
+ self.visit(node.expr)
+ if node.flags == 'OP_ASSIGN':
+ self.emit('STORE_ATTR', self.mangle(node.attrname))
+ elif node.flags == 'OP_DELETE':
+ self.emit('DELETE_ATTR', self.mangle(node.attrname))
+ else:
+ print "warning: unexpected flags:", node.flags
+ print node
+
+ def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
+ if findOp(node) != 'OP_DELETE':
+ self.emit(op, len(node.nodes))
+ for child in node.nodes:
+ self.visit(child)
+
+ if VERSION > 1:
+ visitAssTuple = _visitAssSequence
+ visitAssList = _visitAssSequence
+ else:
+ def visitAssTuple(self, node):
+ self._visitAssSequence(node, 'UNPACK_TUPLE')
+
+ def visitAssList(self, node):
+ self._visitAssSequence(node, 'UNPACK_LIST')
+
+ # augmented assignment
+
+ def visitAugAssign(self, node):
+ self.set_lineno(node)
+ aug_node = wrap_aug(node.node)
+ self.visit(aug_node, "load")
+ self.visit(node.expr)
+ self.emit(self._augmented_opcode[node.op])
+ self.visit(aug_node, "store")
+
+ _augmented_opcode = {
+ '+=' : 'INPLACE_ADD',
+ '-=' : 'INPLACE_SUBTRACT',
+ '*=' : 'INPLACE_MULTIPLY',
+ '/=' : 'INPLACE_DIVIDE',
+ '//=': 'INPLACE_FLOOR_DIVIDE',
+ '%=' : 'INPLACE_MODULO',
+ '**=': 'INPLACE_POWER',
+ '>>=': 'INPLACE_RSHIFT',
+ '<<=': 'INPLACE_LSHIFT',
+ '&=' : 'INPLACE_AND',
+ '^=' : 'INPLACE_XOR',
+ '|=' : 'INPLACE_OR',
+ }
+
+ def visitAugName(self, node, mode):
+ if mode == "load":
+ self.loadName(node.name)
+ elif mode == "store":
+ self.storeName(node.name)
+
+ def visitAugGetattr(self, node, mode):
+ if mode == "load":
+ self.visit(node.expr)
+ self.emit('DUP_TOP')
+ self.emit('LOAD_ATTR', self.mangle(node.attrname))
+ elif mode == "store":
+ self.emit('ROT_TWO')
+ self.emit('STORE_ATTR', self.mangle(node.attrname))
+
+ def visitAugSlice(self, node, mode):
+ if mode == "load":
+ self.visitSlice(node, 1)
+ elif mode == "store":
+ slice = 0
+ if node.lower:
+ slice = slice | 1
+ if node.upper:
+ slice = slice | 2
+ if slice == 0:
+ self.emit('ROT_TWO')
+ elif slice == 3:
+ self.emit('ROT_FOUR')
+ else:
+ self.emit('ROT_THREE')
+ self.emit('STORE_SLICE+%d' % slice)
+
+ def visitAugSubscript(self, node, mode):
+ if mode == "load":
+ self.visitSubscript(node, 1)
+ elif mode == "store":
+ self.emit('ROT_THREE')
+ self.emit('STORE_SUBSCR')
+
+ def visitExec(self, node):
+ self.visit(node.expr)
+ if node.locals is None:
+ self.emit('LOAD_CONST', None)
+ else:
+ self.visit(node.locals)
+ if node.globals is None:
+ self.emit('DUP_TOP')
+ else:
+ self.visit(node.globals)
+ self.emit('EXEC_STMT')
+
+ def visitCallFunc(self, node):
+ pos = 0
+ kw = 0
+ self.set_lineno(node)
+ self.visit(node.node)
+ for arg in node.args:
+ self.visit(arg)
+ if isinstance(arg, ast.Keyword):
+ kw = kw + 1
+ else:
+ pos = pos + 1
+ if node.star_args is not None:
+ self.visit(node.star_args)
+ if node.dstar_args is not None:
+ self.visit(node.dstar_args)
+ have_star = node.star_args is not None
+ have_dstar = node.dstar_args is not None
+ opcode = callfunc_opcode_info[have_star, have_dstar]
+ self.emit(opcode, kw << 8 | pos)
+
+ def visitPrint(self, node, newline=0):
+ self.set_lineno(node)
+ if node.dest:
+ self.visit(node.dest)
+ for child in node.nodes:
+ if node.dest:
+ self.emit('DUP_TOP')
+ self.visit(child)
+ if node.dest:
+ self.emit('ROT_TWO')
+ self.emit('PRINT_ITEM_TO')
+ else:
+ self.emit('PRINT_ITEM')
+ if node.dest and not newline:
+ self.emit('POP_TOP')
+
+ def visitPrintnl(self, node):
+ self.visitPrint(node, newline=1)
+ if node.dest:
+ self.emit('PRINT_NEWLINE_TO')
+ else:
+ self.emit('PRINT_NEWLINE')
+
+ def visitReturn(self, node):
+ self.set_lineno(node)
+ self.visit(node.value)
+ self.emit('RETURN_VALUE')
+
+ def visitYield(self, node):
+ self.set_lineno(node)
+ self.visit(node.value)
+ self.emit('YIELD_VALUE')
+
+ # slice and subscript stuff
+
+ def visitSlice(self, node, aug_flag=None):
+ # aug_flag is used by visitAugSlice
+ self.visit(node.expr)
+ slice = 0
+ if node.lower:
+ self.visit(node.lower)
+ slice = slice | 1
+ if node.upper:
+ self.visit(node.upper)
+ slice = slice | 2
+ if aug_flag:
+ if slice == 0:
+ self.emit('DUP_TOP')
+ elif slice == 3:
+ self.emit('DUP_TOPX', 3)
+ else:
+ self.emit('DUP_TOPX', 2)
+ if node.flags == 'OP_APPLY':
+ self.emit('SLICE+%d' % slice)
+ elif node.flags == 'OP_ASSIGN':
+ self.emit('STORE_SLICE+%d' % slice)
+ elif node.flags == 'OP_DELETE':
+ self.emit('DELETE_SLICE+%d' % slice)
+ else:
+ print "weird slice", node.flags
+ raise
+
+ def visitSubscript(self, node, aug_flag=None):
+ self.visit(node.expr)
+ for sub in node.subs:
+ self.visit(sub)
+ if len(node.subs) > 1:
+ self.emit('BUILD_TUPLE', len(node.subs))
+ if aug_flag:
+ self.emit('DUP_TOPX', 2)
+ if node.flags == 'OP_APPLY':
+ self.emit('BINARY_SUBSCR')
+ elif node.flags == 'OP_ASSIGN':
+ self.emit('STORE_SUBSCR')
+ elif node.flags == 'OP_DELETE':
+ self.emit('DELETE_SUBSCR')
+
+ # binary ops
+
+ def binaryOp(self, node, op):
+ self.visit(node.left)
+ self.visit(node.right)
+ self.emit(op)
+
+ def visitAdd(self, node):
+ return self.binaryOp(node, 'BINARY_ADD')
+
+ def visitSub(self, node):
+ return self.binaryOp(node, 'BINARY_SUBTRACT')
+
+ def visitMul(self, node):
+ return self.binaryOp(node, 'BINARY_MULTIPLY')
+
+ def visitDiv(self, node):
+ return self.binaryOp(node, self._div_op)
+
+ def visitFloorDiv(self, node):
+ return self.binaryOp(node, 'BINARY_FLOOR_DIVIDE')
+
+ def visitMod(self, node):
+ return self.binaryOp(node, 'BINARY_MODULO')
+
+ def visitPower(self, node):
+ return self.binaryOp(node, 'BINARY_POWER')
+
+ def visitLeftShift(self, node):
+ return self.binaryOp(node, 'BINARY_LSHIFT')
+
+ def visitRightShift(self, node):
+ return self.binaryOp(node, 'BINARY_RSHIFT')
+
+ # unary ops
+
+ def unaryOp(self, node, op):
+ self.visit(node.expr)
+ self.emit(op)
+
+ def visitInvert(self, node):
+ return self.unaryOp(node, 'UNARY_INVERT')
+
+ def visitUnarySub(self, node):
+ return self.unaryOp(node, 'UNARY_NEGATIVE')
+
+ def visitUnaryAdd(self, node):
+ return self.unaryOp(node, 'UNARY_POSITIVE')
+
+ def visitUnaryInvert(self, node):
+ return self.unaryOp(node, 'UNARY_INVERT')
+
+ def visitNot(self, node):
+ return self.unaryOp(node, 'UNARY_NOT')
+
+ def visitBackquote(self, node):
+ return self.unaryOp(node, 'UNARY_CONVERT')
+
+ # bit ops
+
+ def bitOp(self, nodes, op):
+ self.visit(nodes[0])
+ for node in nodes[1:]:
+ self.visit(node)
+ self.emit(op)
+
+ def visitBitand(self, node):
+ return self.bitOp(node.nodes, 'BINARY_AND')
+
+ def visitBitor(self, node):
+ return self.bitOp(node.nodes, 'BINARY_OR')
+
+ def visitBitxor(self, node):
+ return self.bitOp(node.nodes, 'BINARY_XOR')
+
+ # object constructors
+
+ def visitEllipsis(self, node):
+ self.emit('LOAD_CONST', Ellipsis)
+
+ def visitTuple(self, node):
+ self.set_lineno(node)
+ for elt in node.nodes:
+ self.visit(elt)
+ self.emit('BUILD_TUPLE', len(node.nodes))
+
+ def visitList(self, node):
+ self.set_lineno(node)
+ for elt in node.nodes:
+ self.visit(elt)
+ self.emit('BUILD_LIST', len(node.nodes))
+
+ def visitSet(self, node):
+ self.set_lineno(node)
+ for elt in node.nodes:
+ self.visit(elt)
+ self.emit('BUILD_SET', len(node.nodes))
+
+ def visitSliceobj(self, node):
+ for child in node.nodes:
+ self.visit(child)
+ self.emit('BUILD_SLICE', len(node.nodes))
+
+ def visitDict(self, node):
+ self.set_lineno(node)
+ self.emit('BUILD_MAP', 0)
+ for k, v in node.items:
+ self.emit('DUP_TOP')
+ self.visit(k)
+ self.visit(v)
+ self.emit('ROT_THREE')
+ self.emit('STORE_SUBSCR')
+
+class NestedScopeMixin:
+ """Defines initClass() for nested scoping (Python 2.2-compatible)"""
+ def initClass(self):
+ self.__class__.NameFinder = LocalNameFinder
+ self.__class__.FunctionGen = FunctionCodeGenerator
+ self.__class__.ClassGen = ClassCodeGenerator
+
+class ModuleCodeGenerator(NestedScopeMixin, CodeGenerator):
+ __super_init = CodeGenerator.__init__
+
+ scopes = None
+
+ def __init__(self, tree):
+ self.graph = pyassem.PyFlowGraph("<module>", tree.filename)
+ self.futures = future.find_futures(tree)
+ self.__super_init()
+ walk(tree, self)
+
+ def get_module(self):
+ return self
+
+class ExpressionCodeGenerator(NestedScopeMixin, CodeGenerator):
+ __super_init = CodeGenerator.__init__
+
+ scopes = None
+ futures = ()
+
+ def __init__(self, tree):
+ self.graph = pyassem.PyFlowGraph("<expression>", tree.filename)
+ self.__super_init()
+ walk(tree, self)
+
+ def get_module(self):
+ return self
+
+class InteractiveCodeGenerator(NestedScopeMixin, CodeGenerator):
+
+ __super_init = CodeGenerator.__init__
+
+ scopes = None
+ futures = ()
+
+ def __init__(self, tree):
+ self.graph = pyassem.PyFlowGraph("<interactive>", tree.filename)
+ self.__super_init()
+ self.set_lineno(tree)
+ walk(tree, self)
+ self.emit('RETURN_VALUE')
+
+ def get_module(self):
+ return self
+
+ def visitDiscard(self, node):
+ # XXX Discard means it's an expression. Perhaps this is a bad
+ # name.
+ self.visit(node.expr)
+ self.emit('PRINT_EXPR')
+
+class AbstractFunctionCode:
+ optimized = 1
+ lambdaCount = 0
+
+ def __init__(self, func, scopes, isLambda, class_name, mod):
+ self.class_name = class_name
+ self.module = mod
+ if isLambda:
+ klass = FunctionCodeGenerator
+ name = "<lambda.%d>" % klass.lambdaCount
+ klass.lambdaCount = klass.lambdaCount + 1
+ else:
+ name = func.name
+
+ args, hasTupleArg = generateArgList(func.argnames)
+ self.graph = pyassem.PyFlowGraph(name, func.filename, args,
+ optimized=1)
+ self.isLambda = isLambda
+ self.super_init()
+
+ if not isLambda and func.doc:
+ self.setDocstring(func.doc)
+
+ lnf = walk(func.code, self.NameFinder(args), verbose=0)
+ self.locals.push(lnf.getLocals())
+ if func.varargs:
+ self.graph.setFlag(CO_VARARGS)
+ if func.kwargs:
+ self.graph.setFlag(CO_VARKEYWORDS)
+ self.set_lineno(func)
+ if hasTupleArg:
+ self.generateArgUnpack(func.argnames)
+
+ def get_module(self):
+ return self.module
+
+ def finish(self):
+ self.graph.startExitBlock()
+ if not self.isLambda:
+ self.emit('LOAD_CONST', None)
+ self.emit('RETURN_VALUE')
+
+ def generateArgUnpack(self, args):
+ for i in range(len(args)):
+ arg = args[i]
+ if isinstance(arg, tuple):
+ self.emit('LOAD_FAST', '.%d' % (i * 2))
+ self.unpackSequence(arg)
+
+ def unpackSequence(self, tup):
+ if VERSION > 1:
+ self.emit('UNPACK_SEQUENCE', len(tup))
+ else:
+ self.emit('UNPACK_TUPLE', len(tup))
+ for elt in tup:
+ if isinstance(elt, tuple):
+ self.unpackSequence(elt)
+ else:
+ self._nameOp('STORE', elt)
+
+ unpackTuple = unpackSequence
+
+class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
+ CodeGenerator):
+ super_init = CodeGenerator.__init__ # call be other init
+ scopes = None
+
+ __super_init = AbstractFunctionCode.__init__
+
+ def __init__(self, func, scopes, isLambda, class_name, mod):
+ self.scopes = scopes
+ self.scope = scopes[func]
+ self.__super_init(func, scopes, isLambda, class_name, mod)
+ self.graph.setFreeVars(self.scope.get_free_vars())
+ self.graph.setCellVars(self.scope.get_cell_vars())
+ if self.scope.generator is not None:
+ self.graph.setFlag(CO_GENERATOR)
+
+class GenExprCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
+ CodeGenerator):
+ super_init = CodeGenerator.__init__ # call be other init
+ scopes = None
+
+ __super_init = AbstractFunctionCode.__init__
+
+ def __init__(self, gexp, scopes, class_name, mod):
+ self.scopes = scopes
+ self.scope = scopes[gexp]
+ self.__super_init(gexp, scopes, 1, class_name, mod)
+ self.graph.setFreeVars(self.scope.get_free_vars())
+ self.graph.setCellVars(self.scope.get_cell_vars())
+ self.graph.setFlag(CO_GENERATOR)
+
+class AbstractClassCode:
+
+ def __init__(self, klass, scopes, module):
+ self.class_name = klass.name
+ self.module = module
+ self.graph = pyassem.PyFlowGraph(klass.name, klass.filename,
+ optimized=0, klass=1)
+ self.super_init()
+ lnf = walk(klass.code, self.NameFinder(), verbose=0)
+ self.locals.push(lnf.getLocals())
+ self.graph.setFlag(CO_NEWLOCALS)
+ if klass.doc:
+ self.setDocstring(klass.doc)
+
+ def get_module(self):
+ return self.module
+
+ def finish(self):
+ self.graph.startExitBlock()
+ self.emit('LOAD_LOCALS')
+ self.emit('RETURN_VALUE')
+
+class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator):
+ super_init = CodeGenerator.__init__
+ scopes = None
+
+ __super_init = AbstractClassCode.__init__
+
+ def __init__(self, klass, scopes, module):
+ self.scopes = scopes
+ self.scope = scopes[klass]
+ self.__super_init(klass, scopes, module)
+ self.graph.setFreeVars(self.scope.get_free_vars())
+ self.graph.setCellVars(self.scope.get_cell_vars())
+ self.set_lineno(klass)
+ self.emit("LOAD_GLOBAL", "__name__")
+ self.storeName("__module__")
+ if klass.doc:
+ self.emit("LOAD_CONST", klass.doc)
+ self.storeName('__doc__')
+
+def generateArgList(arglist):
+ """Generate an arg list marking TupleArgs"""
+ args = []
+ extra = []
+ count = 0
+ for i in range(len(arglist)):
+ elt = arglist[i]
+ if isinstance(elt, str):
+ args.append(elt)
+ elif isinstance(elt, tuple):
+ args.append(TupleArg(i * 2, elt))
+ extra.extend(misc.flatten(elt))
+ count = count + 1
+ else:
+ raise ValueError, "unexpect argument type:", elt
+ return args + extra, count
+
+def findOp(node):
+ """Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
+ v = OpFinder()
+ walk(node, v, verbose=0)
+ return v.op
+
+class OpFinder:
+ def __init__(self):
+ self.op = None
+ def visitAssName(self, node):
+ if self.op is None:
+ self.op = node.flags
+ elif self.op != node.flags:
+ raise ValueError, "mixed ops in stmt"
+ visitAssAttr = visitAssName
+ visitSubscript = visitAssName
+
+class Delegator:
+ """Base class to support delegation for augmented assignment nodes
+
+ To generator code for augmented assignments, we use the following
+ wrapper classes. In visitAugAssign, the left-hand expression node
+ is visited twice. The first time the visit uses the normal method
+ for that node . The second time the visit uses a different method
+ that generates the appropriate code to perform the assignment.
+ These delegator classes wrap the original AST nodes in order to
+ support the variant visit methods.
+ """
+ def __init__(self, obj):
+ self.obj = obj
+
+ def __getattr__(self, attr):
+ return getattr(self.obj, attr)
+
+class AugGetattr(Delegator):
+ pass
+
+class AugName(Delegator):
+ pass
+
+class AugSlice(Delegator):
+ pass
+
+class AugSubscript(Delegator):
+ pass
+
+wrapper = {
+ ast.Getattr: AugGetattr,
+ ast.Name: AugName,
+ ast.Slice: AugSlice,
+ ast.Subscript: AugSubscript,
+ }
+
+def wrap_aug(node):
+ return wrapper[node.__class__](node)
+
+if __name__ == "__main__":
+ for file in sys.argv[1:]:
+ compileFile(file)
diff --git a/lib/python2.7/compiler/symbols.py b/lib/python2.7/compiler/symbols.py
new file mode 100644
index 0000000..afeec50
--- /dev/null
+++ b/lib/python2.7/compiler/symbols.py
@@ -0,0 +1,462 @@
+"""Module symbol-table generator"""
+
+from compiler import ast
+from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \
+ SC_FREE, SC_CELL, SC_UNKNOWN
+from compiler.misc import mangle
+import types
+
+
+import sys
+
+MANGLE_LEN = 256
+
+class Scope:
+ # XXX how much information do I need about each name?
+ def __init__(self, name, module, klass=None):
+ self.name = name
+ self.module = module
+ self.defs = {}
+ self.uses = {}
+ self.globals = {}
+ self.params = {}
+ self.frees = {}
+ self.cells = {}
+ self.children = []
+ # nested is true if the class could contain free variables,
+ # i.e. if it is nested within another function.
+ self.nested = None
+ self.generator = None
+ self.klass = None
+ if klass is not None:
+ for i in range(len(klass)):
+ if klass[i] != '_':
+ self.klass = klass[i:]
+ break
+
+ def __repr__(self):
+ return "<%s: %s>" % (self.__class__.__name__, self.name)
+
+ def mangle(self, name):
+ if self.klass is None:
+ return name
+ return mangle(name, self.klass)
+
+ def add_def(self, name):
+ self.defs[self.mangle(name)] = 1
+
+ def add_use(self, name):
+ self.uses[self.mangle(name)] = 1
+
+ def add_global(self, name):
+ name = self.mangle(name)
+ if name in self.uses or name in self.defs:
+ pass # XXX warn about global following def/use
+ if name in self.params:
+ raise SyntaxError, "%s in %s is global and parameter" % \
+ (name, self.name)
+ self.globals[name] = 1
+ self.module.add_def(name)
+
+ def add_param(self, name):
+ name = self.mangle(name)
+ self.defs[name] = 1
+ self.params[name] = 1
+
+ def get_names(self):
+ d = {}
+ d.update(self.defs)
+ d.update(self.uses)
+ d.update(self.globals)
+ return d.keys()
+
+ def add_child(self, child):
+ self.children.append(child)
+
+ def get_children(self):
+ return self.children
+
+ def DEBUG(self):
+ print >> sys.stderr, self.name, self.nested and "nested" or ""
+ print >> sys.stderr, "\tglobals: ", self.globals
+ print >> sys.stderr, "\tcells: ", self.cells
+ print >> sys.stderr, "\tdefs: ", self.defs
+ print >> sys.stderr, "\tuses: ", self.uses
+ print >> sys.stderr, "\tfrees:", self.frees
+
+ def check_name(self, name):
+ """Return scope of name.
+
+ The scope of a name could be LOCAL, GLOBAL, FREE, or CELL.
+ """
+ if name in self.globals:
+ return SC_GLOBAL_EXPLICIT
+ if name in self.cells:
+ return SC_CELL
+ if name in self.defs:
+ return SC_LOCAL
+ if self.nested and (name in self.frees or name in self.uses):
+ return SC_FREE
+ if self.nested:
+ return SC_UNKNOWN
+ else:
+ return SC_GLOBAL_IMPLICIT
+
+ def get_free_vars(self):
+ if not self.nested:
+ return ()
+ free = {}
+ free.update(self.frees)
+ for name in self.uses.keys():
+ if name not in self.defs and name not in self.globals:
+ free[name] = 1
+ return free.keys()
+
+ def handle_children(self):
+ for child in self.children:
+ frees = child.get_free_vars()
+ globals = self.add_frees(frees)
+ for name in globals:
+ child.force_global(name)
+
+ def force_global(self, name):
+ """Force name to be global in scope.
+
+ Some child of the current node had a free reference to name.
+ When the child was processed, it was labelled a free
+ variable. Now that all its enclosing scope have been
+ processed, the name is known to be a global or builtin. So
+ walk back down the child chain and set the name to be global
+ rather than free.
+
+ Be careful to stop if a child does not think the name is
+ free.
+ """
+ self.globals[name] = 1
+ if name in self.frees:
+ del self.frees[name]
+ for child in self.children:
+ if child.check_name(name) == SC_FREE:
+ child.force_global(name)
+
+ def add_frees(self, names):
+ """Process list of free vars from nested scope.
+
+ Returns a list of names that are either 1) declared global in the
+ parent or 2) undefined in a top-level parent. In either case,
+ the nested scope should treat them as globals.
+ """
+ child_globals = []
+ for name in names:
+ sc = self.check_name(name)
+ if self.nested:
+ if sc == SC_UNKNOWN or sc == SC_FREE \
+ or isinstance(self, ClassScope):
+ self.frees[name] = 1
+ elif sc == SC_GLOBAL_IMPLICIT:
+ child_globals.append(name)
+ elif isinstance(self, FunctionScope) and sc == SC_LOCAL:
+ self.cells[name] = 1
+ elif sc != SC_CELL:
+ child_globals.append(name)
+ else:
+ if sc == SC_LOCAL:
+ self.cells[name] = 1
+ elif sc != SC_CELL:
+ child_globals.append(name)
+ return child_globals
+
+ def get_cell_vars(self):
+ return self.cells.keys()
+
+class ModuleScope(Scope):
+ __super_init = Scope.__init__
+
+ def __init__(self):
+ self.__super_init("global", self)
+
+class FunctionScope(Scope):
+ pass
+
+class GenExprScope(Scope):
+ __super_init = Scope.__init__
+
+ __counter = 1
+
+ def __init__(self, module, klass=None):
+ i = self.__counter
+ self.__counter += 1
+ self.__super_init("generator expression<%d>"%i, module, klass)
+ self.add_param('.0')
+
+ def get_names(self):
+ keys = Scope.get_names(self)
+ return keys
+
+class LambdaScope(FunctionScope):
+ __super_init = Scope.__init__
+
+ __counter = 1
+
+ def __init__(self, module, klass=None):
+ i = self.__counter
+ self.__counter += 1
+ self.__super_init("lambda.%d" % i, module, klass)
+
+class ClassScope(Scope):
+ __super_init = Scope.__init__
+
+ def __init__(self, name, module):
+ self.__super_init(name, module, name)
+
+class SymbolVisitor:
+ def __init__(self):
+ self.scopes = {}
+ self.klass = None
+
+ # node that define new scopes
+
+ def visitModule(self, node):
+ scope = self.module = self.scopes[node] = ModuleScope()
+ self.visit(node.node, scope)
+
+ visitExpression = visitModule
+
+ def visitFunction(self, node, parent):
+ if node.decorators:
+ self.visit(node.decorators, parent)
+ parent.add_def(node.name)
+ for n in node.defaults:
+ self.visit(n, parent)
+ scope = FunctionScope(node.name, self.module, self.klass)
+ if parent.nested or isinstance(parent, FunctionScope):
+ scope.nested = 1
+ self.scopes[node] = scope
+ self._do_args(scope, node.argnames)
+ self.visit(node.code, scope)
+ self.handle_free_vars(scope, parent)
+
+ def visitGenExpr(self, node, parent):
+ scope = GenExprScope(self.module, self.klass);
+ if parent.nested or isinstance(parent, FunctionScope) \
+ or isinstance(parent, GenExprScope):
+ scope.nested = 1
+
+ self.scopes[node] = scope
+ self.visit(node.code, scope)
+
+ self.handle_free_vars(scope, parent)
+
+ def visitGenExprInner(self, node, scope):
+ for genfor in node.quals:
+ self.visit(genfor, scope)
+
+ self.visit(node.expr, scope)
+
+ def visitGenExprFor(self, node, scope):
+ self.visit(node.assign, scope, 1)
+ self.visit(node.iter, scope)
+ for if_ in node.ifs:
+ self.visit(if_, scope)
+
+ def visitGenExprIf(self, node, scope):
+ self.visit(node.test, scope)
+
+ def visitLambda(self, node, parent, assign=0):
+ # Lambda is an expression, so it could appear in an expression
+ # context where assign is passed. The transformer should catch
+ # any code that has a lambda on the left-hand side.
+ assert not assign
+
+ for n in node.defaults:
+ self.visit(n, parent)
+ scope = LambdaScope(self.module, self.klass)
+ if parent.nested or isinstance(parent, FunctionScope):
+ scope.nested = 1
+ self.scopes[node] = scope
+ self._do_args(scope, node.argnames)
+ self.visit(node.code, scope)
+ self.handle_free_vars(scope, parent)
+
+ def _do_args(self, scope, args):
+ for name in args:
+ if type(name) == types.TupleType:
+ self._do_args(scope, name)
+ else:
+ scope.add_param(name)
+
+ def handle_free_vars(self, scope, parent):
+ parent.add_child(scope)
+ scope.handle_children()
+
+ def visitClass(self, node, parent):
+ parent.add_def(node.name)
+ for n in node.bases:
+ self.visit(n, parent)
+ scope = ClassScope(node.name, self.module)
+ if parent.nested or isinstance(parent, FunctionScope):
+ scope.nested = 1
+ if node.doc is not None:
+ scope.add_def('__doc__')
+ scope.add_def('__module__')
+ self.scopes[node] = scope
+ prev = self.klass
+ self.klass = node.name
+ self.visit(node.code, scope)
+ self.klass = prev
+ self.handle_free_vars(scope, parent)
+
+ # name can be a def or a use
+
+ # XXX a few calls and nodes expect a third "assign" arg that is
+ # true if the name is being used as an assignment. only
+ # expressions contained within statements may have the assign arg.
+
+ def visitName(self, node, scope, assign=0):
+ if assign:
+ scope.add_def(node.name)
+ else:
+ scope.add_use(node.name)
+
+ # operations that bind new names
+
+ def visitFor(self, node, scope):
+ self.visit(node.assign, scope, 1)
+ self.visit(node.list, scope)
+ self.visit(node.body, scope)
+ if node.else_:
+ self.visit(node.else_, scope)
+
+ def visitFrom(self, node, scope):
+ for name, asname in node.names:
+ if name == "*":
+ continue
+ scope.add_def(asname or name)
+
+ def visitImport(self, node, scope):
+ for name, asname in node.names:
+ i = name.find(".")
+ if i > -1:
+ name = name[:i]
+ scope.add_def(asname or name)
+
+ def visitGlobal(self, node, scope):
+ for name in node.names:
+ scope.add_global(name)
+
+ def visitAssign(self, node, scope):
+ """Propagate assignment flag down to child nodes.
+
+ The Assign node doesn't itself contains the variables being
+ assigned to. Instead, the children in node.nodes are visited
+ with the assign flag set to true. When the names occur in
+ those nodes, they are marked as defs.
+
+ Some names that occur in an assignment target are not bound by
+ the assignment, e.g. a name occurring inside a slice. The
+ visitor handles these nodes specially; they do not propagate
+ the assign flag to their children.
+ """
+ for n in node.nodes:
+ self.visit(n, scope, 1)
+ self.visit(node.expr, scope)
+
+ def visitAssName(self, node, scope, assign=1):
+ scope.add_def(node.name)
+
+ def visitAssAttr(self, node, scope, assign=0):
+ self.visit(node.expr, scope, 0)
+
+ def visitSubscript(self, node, scope, assign=0):
+ self.visit(node.expr, scope, 0)
+ for n in node.subs:
+ self.visit(n, scope, 0)
+
+ def visitSlice(self, node, scope, assign=0):
+ self.visit(node.expr, scope, 0)
+ if node.lower:
+ self.visit(node.lower, scope, 0)
+ if node.upper:
+ self.visit(node.upper, scope, 0)
+
+ def visitAugAssign(self, node, scope):
+ # If the LHS is a name, then this counts as assignment.
+ # Otherwise, it's just use.
+ self.visit(node.node, scope)
+ if isinstance(node.node, ast.Name):
+ self.visit(node.node, scope, 1) # XXX worry about this
+ self.visit(node.expr, scope)
+
+ # prune if statements if tests are false
+
+ _const_types = types.StringType, types.IntType, types.FloatType
+
+ def visitIf(self, node, scope):
+ for test, body in node.tests:
+ if isinstance(test, ast.Const):
+ if type(test.value) in self._const_types:
+ if not test.value:
+ continue
+ self.visit(test, scope)
+ self.visit(body, scope)
+ if node.else_:
+ self.visit(node.else_, scope)
+
+ # a yield statement signals a generator
+
+ def visitYield(self, node, scope):
+ scope.generator = 1
+ self.visit(node.value, scope)
+
+def list_eq(l1, l2):
+ return sorted(l1) == sorted(l2)
+
+if __name__ == "__main__":
+ import sys
+ from compiler import parseFile, walk
+ import symtable
+
+ def get_names(syms):
+ return [s for s in [s.get_name() for s in syms.get_symbols()]
+ if not (s.startswith('_[') or s.startswith('.'))]
+
+ for file in sys.argv[1:]:
+ print file
+ f = open(file)
+ buf = f.read()
+ f.close()
+ syms = symtable.symtable(buf, file, "exec")
+ mod_names = get_names(syms)
+ tree = parseFile(file)
+ s = SymbolVisitor()
+ walk(tree, s)
+
+ # compare module-level symbols
+ names2 = s.scopes[tree].get_names()
+
+ if not list_eq(mod_names, names2):
+ print
+ print "oops", file
+ print sorted(mod_names)
+ print sorted(names2)
+ sys.exit(-1)
+
+ d = {}
+ d.update(s.scopes)
+ del d[tree]
+ scopes = d.values()
+ del d
+
+ for s in syms.get_symbols():
+ if s.is_namespace():
+ l = [sc for sc in scopes
+ if sc.name == s.get_name()]
+ if len(l) > 1:
+ print "skipping", s.get_name()
+ else:
+ if not list_eq(get_names(s.get_namespace()),
+ l[0].get_names()):
+ print s.get_name()
+ print sorted(get_names(s.get_namespace()))
+ print sorted(l[0].get_names())
+ sys.exit(-1)
diff --git a/lib/python2.7/compiler/syntax.py b/lib/python2.7/compiler/syntax.py
new file mode 100644
index 0000000..a45d9c2
--- /dev/null
+++ b/lib/python2.7/compiler/syntax.py
@@ -0,0 +1,46 @@
+"""Check for errs in the AST.
+
+The Python parser does not catch all syntax errors. Others, like
+assignments with invalid targets, are caught in the code generation
+phase.
+
+The compiler package catches some errors in the transformer module.
+But it seems clearer to write checkers that use the AST to detect
+errors.
+"""
+
+from compiler import ast, walk
+
+def check(tree, multi=None):
+ v = SyntaxErrorChecker(multi)
+ walk(tree, v)
+ return v.errors
+
+class SyntaxErrorChecker:
+ """A visitor to find syntax errors in the AST."""
+
+ def __init__(self, multi=None):
+ """Create new visitor object.
+
+ If optional argument multi is not None, then print messages
+ for each error rather than raising a SyntaxError for the
+ first.
+ """
+ self.multi = multi
+ self.errors = 0
+
+ def error(self, node, msg):
+ self.errors = self.errors + 1
+ if self.multi is not None:
+ print "%s:%s: %s" % (node.filename, node.lineno, msg)
+ else:
+ raise SyntaxError, "%s (%s:%s)" % (msg, node.filename, node.lineno)
+
+ def visitAssign(self, node):
+ # the transformer module handles many of these
+ pass
+## for target in node.nodes:
+## if isinstance(target, ast.AssList):
+## if target.lineno is None:
+## target.lineno = node.lineno
+## self.error(target, "can't assign to list comprehension")
diff --git a/lib/python2.7/compiler/transformer.py b/lib/python2.7/compiler/transformer.py
new file mode 100644
index 0000000..d4f4613
--- /dev/null
+++ b/lib/python2.7/compiler/transformer.py
@@ -0,0 +1,1535 @@
+"""Parse tree transformation module.
+
+Transforms Python source code into an abstract syntax tree (AST)
+defined in the ast module.
+
+The simplest ways to invoke this module are via parse and parseFile.
+parse(buf) -> AST
+parseFile(path) -> AST
+"""
+
+# Original version written by Greg Stein (gstein@lyra.org)
+# and Bill Tutt (rassilon@lima.mudlib.org)
+# February 1997.
+#
+# Modifications and improvements for Python 2.0 by Jeremy Hylton and
+# Mark Hammond
+#
+# Some fixes to try to have correct line number on almost all nodes
+# (except Module, Discard and Stmt) added by Sylvain Thenault
+#
+# Portions of this file are:
+# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
+#
+# This module is provided under a BSD-ish license. See
+# http://www.opensource.org/licenses/bsd-license.html
+# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
+
+from compiler.ast import *
+import parser
+import symbol
+import token
+
+class WalkerError(StandardError):
+ pass
+
+from compiler.consts import CO_VARARGS, CO_VARKEYWORDS
+from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY
+
+def parseFile(path):
+ f = open(path, "U")
+ # XXX The parser API tolerates files without a trailing newline,
+ # but not strings without a trailing newline. Always add an extra
+ # newline to the file contents, since we're going through the string
+ # version of the API.
+ src = f.read() + "\n"
+ f.close()
+ return parse(src)
+
+def parse(buf, mode="exec"):
+ if mode == "exec" or mode == "single":
+ return Transformer().parsesuite(buf)
+ elif mode == "eval":
+ return Transformer().parseexpr(buf)
+ else:
+ raise ValueError("compile() arg 3 must be"
+ " 'exec' or 'eval' or 'single'")
+
+def asList(nodes):
+ l = []
+ for item in nodes:
+ if hasattr(item, "asList"):
+ l.append(item.asList())
+ else:
+ if type(item) is type( (None, None) ):
+ l.append(tuple(asList(item)))
+ elif type(item) is type( [] ):
+ l.append(asList(item))
+ else:
+ l.append(item)
+ return l
+
+def extractLineNo(ast):
+ if not isinstance(ast[1], tuple):
+ # get a terminal node
+ return ast[2]
+ for child in ast[1:]:
+ if isinstance(child, tuple):
+ lineno = extractLineNo(child)
+ if lineno is not None:
+ return lineno
+
+def Node(*args):
+ kind = args[0]
+ if kind in nodes:
+ try:
+ return nodes[kind](*args[1:])
+ except TypeError:
+ print nodes[kind], len(args), args
+ raise
+ else:
+ raise WalkerError, "Can't find appropriate Node type: %s" % str(args)
+ #return apply(ast.Node, args)
+
+class Transformer:
+ """Utility object for transforming Python parse trees.
+
+ Exposes the following methods:
+ tree = transform(ast_tree)
+ tree = parsesuite(text)
+ tree = parseexpr(text)
+ tree = parsefile(fileob | filename)
+ """
+
+ def __init__(self):
+ self._dispatch = {}
+ for value, name in symbol.sym_name.items():
+ if hasattr(self, name):
+ self._dispatch[value] = getattr(self, name)
+ self._dispatch[token.NEWLINE] = self.com_NEWLINE
+ self._atom_dispatch = {token.LPAR: self.atom_lpar,
+ token.LSQB: self.atom_lsqb,
+ token.LBRACE: self.atom_lbrace,
+ token.BACKQUOTE: self.atom_backquote,
+ token.NUMBER: self.atom_number,
+ token.STRING: self.atom_string,
+ token.NAME: self.atom_name,
+ }
+ self.encoding = None
+
+ def transform(self, tree):
+ """Transform an AST into a modified parse tree."""
+ if not (isinstance(tree, tuple) or isinstance(tree, list)):
+ tree = parser.st2tuple(tree, line_info=1)
+ return self.compile_node(tree)
+
+ def parsesuite(self, text):
+ """Return a modified parse tree for the given suite text."""
+ return self.transform(parser.suite(text))
+
+ def parseexpr(self, text):
+ """Return a modified parse tree for the given expression text."""
+ return self.transform(parser.expr(text))
+
+ def parsefile(self, file):
+ """Return a modified parse tree for the contents of the given file."""
+ if type(file) == type(''):
+ file = open(file)
+ return self.parsesuite(file.read())
+
+ # --------------------------------------------------------------
+ #
+ # PRIVATE METHODS
+ #
+
+ def compile_node(self, node):
+ ### emit a line-number node?
+ n = node[0]
+
+ if n == symbol.encoding_decl:
+ self.encoding = node[2]
+ node = node[1]
+ n = node[0]
+
+ if n == symbol.single_input:
+ return self.single_input(node[1:])
+ if n == symbol.file_input:
+ return self.file_input(node[1:])
+ if n == symbol.eval_input:
+ return self.eval_input(node[1:])
+ if n == symbol.lambdef:
+ return self.lambdef(node[1:])
+ if n == symbol.funcdef:
+ return self.funcdef(node[1:])
+ if n == symbol.classdef:
+ return self.classdef(node[1:])
+
+ raise WalkerError, ('unexpected node type', n)
+
+ def single_input(self, node):
+ ### do we want to do anything about being "interactive" ?
+
+ # NEWLINE | simple_stmt | compound_stmt NEWLINE
+ n = node[0][0]
+ if n != token.NEWLINE:
+ return self.com_stmt(node[0])
+
+ return Pass()
+
+ def file_input(self, nodelist):
+ doc = self.get_docstring(nodelist, symbol.file_input)
+ if doc is not None:
+ i = 1
+ else:
+ i = 0
+ stmts = []
+ for node in nodelist[i:]:
+ if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
+ self.com_append_stmt(stmts, node)
+ return Module(doc, Stmt(stmts))
+
+ def eval_input(self, nodelist):
+ # from the built-in function input()
+ ### is this sufficient?
+ return Expression(self.com_node(nodelist[0]))
+
+ def decorator_name(self, nodelist):
+ listlen = len(nodelist)
+ assert listlen >= 1 and listlen % 2 == 1
+
+ item = self.atom_name(nodelist)
+ i = 1
+ while i < listlen:
+ assert nodelist[i][0] == token.DOT
+ assert nodelist[i + 1][0] == token.NAME
+ item = Getattr(item, nodelist[i + 1][1])
+ i += 2
+
+ return item
+
+ def decorator(self, nodelist):
+ # '@' dotted_name [ '(' [arglist] ')' ]
+ assert len(nodelist) in (3, 5, 6)
+ assert nodelist[0][0] == token.AT
+ assert nodelist[-1][0] == token.NEWLINE
+
+ assert nodelist[1][0] == symbol.dotted_name
+ funcname = self.decorator_name(nodelist[1][1:])
+
+ if len(nodelist) > 3:
+ assert nodelist[2][0] == token.LPAR
+ expr = self.com_call_function(funcname, nodelist[3])
+ else:
+ expr = funcname
+
+ return expr
+
+ def decorators(self, nodelist):
+ # decorators: decorator ([NEWLINE] decorator)* NEWLINE
+ items = []
+ for dec_nodelist in nodelist:
+ assert dec_nodelist[0] == symbol.decorator
+ items.append(self.decorator(dec_nodelist[1:]))
+ return Decorators(items)
+
+ def decorated(self, nodelist):
+ assert nodelist[0][0] == symbol.decorators
+ if nodelist[1][0] == symbol.funcdef:
+ n = [nodelist[0]] + list(nodelist[1][1:])
+ return self.funcdef(n)
+ elif nodelist[1][0] == symbol.classdef:
+ decorators = self.decorators(nodelist[0][1:])
+ cls = self.classdef(nodelist[1][1:])
+ cls.decorators = decorators
+ return cls
+ raise WalkerError()
+
+ def funcdef(self, nodelist):
+ # -6 -5 -4 -3 -2 -1
+ # funcdef: [decorators] 'def' NAME parameters ':' suite
+ # parameters: '(' [varargslist] ')'
+
+ if len(nodelist) == 6:
+ assert nodelist[0][0] == symbol.decorators
+ decorators = self.decorators(nodelist[0][1:])
+ else:
+ assert len(nodelist) == 5
+ decorators = None
+
+ lineno = nodelist[-4][2]
+ name = nodelist[-4][1]
+ args = nodelist[-3][2]
+
+ if args[0] == symbol.varargslist:
+ names, defaults, flags = self.com_arglist(args[1:])
+ else:
+ names = defaults = ()
+ flags = 0
+ doc = self.get_docstring(nodelist[-1])
+
+ # code for function
+ code = self.com_node(nodelist[-1])
+
+ if doc is not None:
+ assert isinstance(code, Stmt)
+ assert isinstance(code.nodes[0], Discard)
+ del code.nodes[0]
+ return Function(decorators, name, names, defaults, flags, doc, code,
+ lineno=lineno)
+
+ def lambdef(self, nodelist):
+ # lambdef: 'lambda' [varargslist] ':' test
+ if nodelist[2][0] == symbol.varargslist:
+ names, defaults, flags = self.com_arglist(nodelist[2][1:])
+ else:
+ names = defaults = ()
+ flags = 0
+
+ # code for lambda
+ code = self.com_node(nodelist[-1])
+
+ return Lambda(names, defaults, flags, code, lineno=nodelist[1][2])
+ old_lambdef = lambdef
+
+ def classdef(self, nodelist):
+ # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
+
+ name = nodelist[1][1]
+ doc = self.get_docstring(nodelist[-1])
+ if nodelist[2][0] == token.COLON:
+ bases = []
+ elif nodelist[3][0] == token.RPAR:
+ bases = []
+ else:
+ bases = self.com_bases(nodelist[3])
+
+ # code for class
+ code = self.com_node(nodelist[-1])
+
+ if doc is not None:
+ assert isinstance(code, Stmt)
+ assert isinstance(code.nodes[0], Discard)
+ del code.nodes[0]
+
+ return Class(name, bases, doc, code, lineno=nodelist[1][2])
+
+ def stmt(self, nodelist):
+ return self.com_stmt(nodelist[0])
+
+ small_stmt = stmt
+ flow_stmt = stmt
+ compound_stmt = stmt
+
+ def simple_stmt(self, nodelist):
+ # small_stmt (';' small_stmt)* [';'] NEWLINE
+ stmts = []
+ for i in range(0, len(nodelist), 2):
+ self.com_append_stmt(stmts, nodelist[i])
+ return Stmt(stmts)
+
+ def parameters(self, nodelist):
+ raise WalkerError
+
+ def varargslist(self, nodelist):
+ raise WalkerError
+
+ def fpdef(self, nodelist):
+ raise WalkerError
+
+ def fplist(self, nodelist):
+ raise WalkerError
+
+ def dotted_name(self, nodelist):
+ raise WalkerError
+
+ def comp_op(self, nodelist):
+ raise WalkerError
+
+ def trailer(self, nodelist):
+ raise WalkerError
+
+ def sliceop(self, nodelist):
+ raise WalkerError
+
+ def argument(self, nodelist):
+ raise WalkerError
+
+ # --------------------------------------------------------------
+ #
+ # STATEMENT NODES (invoked by com_node())
+ #
+
+ def expr_stmt(self, nodelist):
+ # augassign testlist | testlist ('=' testlist)*
+ en = nodelist[-1]
+ exprNode = self.lookup_node(en)(en[1:])
+ if len(nodelist) == 1:
+ return Discard(exprNode, lineno=exprNode.lineno)
+ if nodelist[1][0] == token.EQUAL:
+ nodesl = []
+ for i in range(0, len(nodelist) - 2, 2):
+ nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
+ return Assign(nodesl, exprNode, lineno=nodelist[1][2])
+ else:
+ lval = self.com_augassign(nodelist[0])
+ op = self.com_augassign_op(nodelist[1])
+ return AugAssign(lval, op[1], exprNode, lineno=op[2])
+ raise WalkerError, "can't get here"
+
+ def print_stmt(self, nodelist):
+ # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
+ items = []
+ if len(nodelist) == 1:
+ start = 1
+ dest = None
+ elif nodelist[1][0] == token.RIGHTSHIFT:
+ assert len(nodelist) == 3 \
+ or nodelist[3][0] == token.COMMA
+ dest = self.com_node(nodelist[2])
+ start = 4
+ else:
+ dest = None
+ start = 1
+ for i in range(start, len(nodelist), 2):
+ items.append(self.com_node(nodelist[i]))
+ if nodelist[-1][0] == token.COMMA:
+ return Print(items, dest, lineno=nodelist[0][2])
+ return Printnl(items, dest, lineno=nodelist[0][2])
+
+ def del_stmt(self, nodelist):
+ return self.com_assign(nodelist[1], OP_DELETE)
+
+ def pass_stmt(self, nodelist):
+ return Pass(lineno=nodelist[0][2])
+
+ def break_stmt(self, nodelist):
+ return Break(lineno=nodelist[0][2])
+
+ def continue_stmt(self, nodelist):
+ return Continue(lineno=nodelist[0][2])
+
+ def return_stmt(self, nodelist):
+ # return: [testlist]
+ if len(nodelist) < 2:
+ return Return(Const(None), lineno=nodelist[0][2])
+ return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
+
+ def yield_stmt(self, nodelist):
+ expr = self.com_node(nodelist[0])
+ return Discard(expr, lineno=expr.lineno)
+
+ def yield_expr(self, nodelist):
+ if len(nodelist) > 1:
+ value = self.com_node(nodelist[1])
+ else:
+ value = Const(None)
+ return Yield(value, lineno=nodelist[0][2])
+
+ def raise_stmt(self, nodelist):
+ # raise: [test [',' test [',' test]]]
+ if len(nodelist) > 5:
+ expr3 = self.com_node(nodelist[5])
+ else:
+ expr3 = None
+ if len(nodelist) > 3:
+ expr2 = self.com_node(nodelist[3])
+ else:
+ expr2 = None
+ if len(nodelist) > 1:
+ expr1 = self.com_node(nodelist[1])
+ else:
+ expr1 = None
+ return Raise(expr1, expr2, expr3, lineno=nodelist[0][2])
+
+ def import_stmt(self, nodelist):
+ # import_stmt: import_name | import_from
+ assert len(nodelist) == 1
+ return self.com_node(nodelist[0])
+
+ def import_name(self, nodelist):
+ # import_name: 'import' dotted_as_names
+ return Import(self.com_dotted_as_names(nodelist[1]),
+ lineno=nodelist[0][2])
+
+ def import_from(self, nodelist):
+ # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' |
+ # '(' import_as_names ')' | import_as_names)
+ assert nodelist[0][1] == 'from'
+ idx = 1
+ while nodelist[idx][1] == '.':
+ idx += 1
+ level = idx - 1
+ if nodelist[idx][0] == symbol.dotted_name:
+ fromname = self.com_dotted_name(nodelist[idx])
+ idx += 1
+ else:
+ fromname = ""
+ assert nodelist[idx][1] == 'import'
+ if nodelist[idx + 1][0] == token.STAR:
+ return From(fromname, [('*', None)], level,
+ lineno=nodelist[0][2])
+ else:
+ node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)]
+ return From(fromname, self.com_import_as_names(node), level,
+ lineno=nodelist[0][2])
+
+ def global_stmt(self, nodelist):
+ # global: NAME (',' NAME)*
+ names = []
+ for i in range(1, len(nodelist), 2):
+ names.append(nodelist[i][1])
+ return Global(names, lineno=nodelist[0][2])
+
+ def exec_stmt(self, nodelist):
+ # exec_stmt: 'exec' expr ['in' expr [',' expr]]
+ expr1 = self.com_node(nodelist[1])
+ if len(nodelist) >= 4:
+ expr2 = self.com_node(nodelist[3])
+ if len(nodelist) >= 6:
+ expr3 = self.com_node(nodelist[5])
+ else:
+ expr3 = None
+ else:
+ expr2 = expr3 = None
+
+ return Exec(expr1, expr2, expr3, lineno=nodelist[0][2])
+
+ def assert_stmt(self, nodelist):
+ # 'assert': test, [',' test]
+ expr1 = self.com_node(nodelist[1])
+ if (len(nodelist) == 4):
+ expr2 = self.com_node(nodelist[3])
+ else:
+ expr2 = None
+ return Assert(expr1, expr2, lineno=nodelist[0][2])
+
+ def if_stmt(self, nodelist):
+ # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+ tests = []
+ for i in range(0, len(nodelist) - 3, 4):
+ testNode = self.com_node(nodelist[i + 1])
+ suiteNode = self.com_node(nodelist[i + 3])
+ tests.append((testNode, suiteNode))
+
+ if len(nodelist) % 4 == 3:
+ elseNode = self.com_node(nodelist[-1])
+## elseNode.lineno = nodelist[-1][1][2]
+ else:
+ elseNode = None
+ return If(tests, elseNode, lineno=nodelist[0][2])
+
+ def while_stmt(self, nodelist):
+ # 'while' test ':' suite ['else' ':' suite]
+
+ testNode = self.com_node(nodelist[1])
+ bodyNode = self.com_node(nodelist[3])
+
+ if len(nodelist) > 4:
+ elseNode = self.com_node(nodelist[6])
+ else:
+ elseNode = None
+
+ return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])
+
+ def for_stmt(self, nodelist):
+ # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+
+ assignNode = self.com_assign(nodelist[1], OP_ASSIGN)
+ listNode = self.com_node(nodelist[3])
+ bodyNode = self.com_node(nodelist[5])
+
+ if len(nodelist) > 8:
+ elseNode = self.com_node(nodelist[8])
+ else:
+ elseNode = None
+
+ return For(assignNode, listNode, bodyNode, elseNode,
+ lineno=nodelist[0][2])
+
+ def try_stmt(self, nodelist):
+ return self.com_try_except_finally(nodelist)
+
+ def with_stmt(self, nodelist):
+ return self.com_with(nodelist)
+
+ def with_var(self, nodelist):
+ return self.com_with_var(nodelist)
+
+ def suite(self, nodelist):
+ # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+ if len(nodelist) == 1:
+ return self.com_stmt(nodelist[0])
+
+ stmts = []
+ for node in nodelist:
+ if node[0] == symbol.stmt:
+ self.com_append_stmt(stmts, node)
+ return Stmt(stmts)
+
+ # --------------------------------------------------------------
+ #
+ # EXPRESSION NODES (invoked by com_node())
+ #
+
+ def testlist(self, nodelist):
+ # testlist: expr (',' expr)* [',']
+ # testlist_safe: test [(',' test)+ [',']]
+ # exprlist: expr (',' expr)* [',']
+ return self.com_binary(Tuple, nodelist)
+
+ testlist_safe = testlist # XXX
+ testlist1 = testlist
+ exprlist = testlist
+
+ def testlist_comp(self, nodelist):
+ # test ( comp_for | (',' test)* [','] )
+ assert nodelist[0][0] == symbol.test
+ if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for:
+ test = self.com_node(nodelist[0])
+ return self.com_generator_expression(test, nodelist[1])
+ return self.testlist(nodelist)
+
+ def test(self, nodelist):
+ # or_test ['if' or_test 'else' test] | lambdef
+ if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
+ return self.lambdef(nodelist[0])
+ then = self.com_node(nodelist[0])
+ if len(nodelist) > 1:
+ assert len(nodelist) == 5
+ assert nodelist[1][1] == 'if'
+ assert nodelist[3][1] == 'else'
+ test = self.com_node(nodelist[2])
+ else_ = self.com_node(nodelist[4])
+ return IfExp(test, then, else_, lineno=nodelist[1][2])
+ return then
+
+ def or_test(self, nodelist):
+ # and_test ('or' and_test)* | lambdef
+ if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
+ return self.lambdef(nodelist[0])
+ return self.com_binary(Or, nodelist)
+ old_test = or_test
+
+ def and_test(self, nodelist):
+ # not_test ('and' not_test)*
+ return self.com_binary(And, nodelist)
+
+ def not_test(self, nodelist):
+ # 'not' not_test | comparison
+ result = self.com_node(nodelist[-1])
+ if len(nodelist) == 2:
+ return Not(result, lineno=nodelist[0][2])
+ return result
+
+ def comparison(self, nodelist):
+ # comparison: expr (comp_op expr)*
+ node = self.com_node(nodelist[0])
+ if len(nodelist) == 1:
+ return node
+
+ results = []
+ for i in range(2, len(nodelist), 2):
+ nl = nodelist[i-1]
+
+ # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
+ # | 'in' | 'not' 'in' | 'is' | 'is' 'not'
+ n = nl[1]
+ if n[0] == token.NAME:
+ type = n[1]
+ if len(nl) == 3:
+ if type == 'not':
+ type = 'not in'
+ else:
+ type = 'is not'
+ else:
+ type = _cmp_types[n[0]]
+
+ lineno = nl[1][2]
+ results.append((type, self.com_node(nodelist[i])))
+
+ # we need a special "compare" node so that we can distinguish
+ # 3 < x < 5 from (3 < x) < 5
+ # the two have very different semantics and results (note that the
+ # latter form is always true)
+
+ return Compare(node, results, lineno=lineno)
+
+ def expr(self, nodelist):
+ # xor_expr ('|' xor_expr)*
+ return self.com_binary(Bitor, nodelist)
+
+ def xor_expr(self, nodelist):
+ # xor_expr ('^' xor_expr)*
+ return self.com_binary(Bitxor, nodelist)
+
+ def and_expr(self, nodelist):
+ # xor_expr ('&' xor_expr)*
+ return self.com_binary(Bitand, nodelist)
+
+ def shift_expr(self, nodelist):
+ # shift_expr ('<<'|'>>' shift_expr)*
+ node = self.com_node(nodelist[0])
+ for i in range(2, len(nodelist), 2):
+ right = self.com_node(nodelist[i])
+ if nodelist[i-1][0] == token.LEFTSHIFT:
+ node = LeftShift([node, right], lineno=nodelist[1][2])
+ elif nodelist[i-1][0] == token.RIGHTSHIFT:
+ node = RightShift([node, right], lineno=nodelist[1][2])
+ else:
+ raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
+ return node
+
+ def arith_expr(self, nodelist):
+ node = self.com_node(nodelist[0])
+ for i in range(2, len(nodelist), 2):
+ right = self.com_node(nodelist[i])
+ if nodelist[i-1][0] == token.PLUS:
+ node = Add([node, right], lineno=nodelist[1][2])
+ elif nodelist[i-1][0] == token.MINUS:
+ node = Sub([node, right], lineno=nodelist[1][2])
+ else:
+ raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
+ return node
+
+ def term(self, nodelist):
+ node = self.com_node(nodelist[0])
+ for i in range(2, len(nodelist), 2):
+ right = self.com_node(nodelist[i])
+ t = nodelist[i-1][0]
+ if t == token.STAR:
+ node = Mul([node, right])
+ elif t == token.SLASH:
+ node = Div([node, right])
+ elif t == token.PERCENT:
+ node = Mod([node, right])
+ elif t == token.DOUBLESLASH:
+ node = FloorDiv([node, right])
+ else:
+ raise ValueError, "unexpected token: %s" % t
+ node.lineno = nodelist[1][2]
+ return node
+
+ def factor(self, nodelist):
+ elt = nodelist[0]
+ t = elt[0]
+ node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
+ # need to handle (unary op)constant here...
+ if t == token.PLUS:
+ return UnaryAdd(node, lineno=elt[2])
+ elif t == token.MINUS:
+ return UnarySub(node, lineno=elt[2])
+ elif t == token.TILDE:
+ node = Invert(node, lineno=elt[2])
+ return node
+
+ def power(self, nodelist):
+ # power: atom trailer* ('**' factor)*
+ node = self.com_node(nodelist[0])
+ for i in range(1, len(nodelist)):
+ elt = nodelist[i]
+ if elt[0] == token.DOUBLESTAR:
+ return Power([node, self.com_node(nodelist[i+1])],
+ lineno=elt[2])
+
+ node = self.com_apply_trailer(node, elt)
+
+ return node
+
+ def atom(self, nodelist):
+ return self._atom_dispatch[nodelist[0][0]](nodelist)
+
+ def atom_lpar(self, nodelist):
+ if nodelist[1][0] == token.RPAR:
+ return Tuple((), lineno=nodelist[0][2])
+ return self.com_node(nodelist[1])
+
+ def atom_lsqb(self, nodelist):
+ if nodelist[1][0] == token.RSQB:
+ return List((), lineno=nodelist[0][2])
+ return self.com_list_constructor(nodelist[1])
+
+ def atom_lbrace(self, nodelist):
+ if nodelist[1][0] == token.RBRACE:
+ return Dict((), lineno=nodelist[0][2])
+ return self.com_dictorsetmaker(nodelist[1])
+
+ def atom_backquote(self, nodelist):
+ return Backquote(self.com_node(nodelist[1]))
+
+ def atom_number(self, nodelist):
+ ### need to verify this matches compile.c
+ k = eval(nodelist[0][1])
+ return Const(k, lineno=nodelist[0][2])
+
+ def decode_literal(self, lit):
+ if self.encoding:
+ # this is particularly fragile & a bit of a
+ # hack... changes in compile.c:parsestr and
+ # tokenizer.c must be reflected here.
+ if self.encoding not in ['utf-8', 'iso-8859-1']:
+ lit = unicode(lit, 'utf-8').encode(self.encoding)
+ return eval("# coding: %s\n%s" % (self.encoding, lit))
+ else:
+ return eval(lit)
+
+ def atom_string(self, nodelist):
+ k = ''
+ for node in nodelist:
+ k += self.decode_literal(node[1])
+ return Const(k, lineno=nodelist[0][2])
+
+ def atom_name(self, nodelist):
+ return Name(nodelist[0][1], lineno=nodelist[0][2])
+
+ # --------------------------------------------------------------
+ #
+ # INTERNAL PARSING UTILITIES
+ #
+
+ # The use of com_node() introduces a lot of extra stack frames,
+ # enough to cause a stack overflow compiling test.test_parser with
+ # the standard interpreter recursionlimit. The com_node() is a
+ # convenience function that hides the dispatch details, but comes
+ # at a very high cost. It is more efficient to dispatch directly
+ # in the callers. In these cases, use lookup_node() and call the
+ # dispatched node directly.
+
+ def lookup_node(self, node):
+ return self._dispatch[node[0]]
+
+ def com_node(self, node):
+ # Note: compile.c has handling in com_node for del_stmt, pass_stmt,
+ # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
+ # and compound_stmt.
+ # We'll just dispatch them.
+ return self._dispatch[node[0]](node[1:])
+
+ def com_NEWLINE(self, *args):
+ # A ';' at the end of a line can make a NEWLINE token appear
+ # here, Render it harmless. (genc discards ('discard',
+ # ('const', xxxx)) Nodes)
+ return Discard(Const(None))
+
+ def com_arglist(self, nodelist):
+ # varargslist:
+ # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME)
+ # | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+ # fpdef: NAME | '(' fplist ')'
+ # fplist: fpdef (',' fpdef)* [',']
+ names = []
+ defaults = []
+ flags = 0
+
+ i = 0
+ while i < len(nodelist):
+ node = nodelist[i]
+ if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
+ if node[0] == token.STAR:
+ node = nodelist[i+1]
+ if node[0] == token.NAME:
+ names.append(node[1])
+ flags = flags | CO_VARARGS
+ i = i + 3
+
+ if i < len(nodelist):
+ # should be DOUBLESTAR
+ t = nodelist[i][0]
+ if t == token.DOUBLESTAR:
+ node = nodelist[i+1]
+ else:
+ raise ValueError, "unexpected token: %s" % t
+ names.append(node[1])
+ flags = flags | CO_VARKEYWORDS
+
+ break
+
+ # fpdef: NAME | '(' fplist ')'
+ names.append(self.com_fpdef(node))
+
+ i = i + 1
+ if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
+ defaults.append(self.com_node(nodelist[i + 1]))
+ i = i + 2
+ elif len(defaults):
+ # we have already seen an argument with default, but here
+ # came one without
+ raise SyntaxError, "non-default argument follows default argument"
+
+ # skip the comma
+ i = i + 1
+
+ return names, defaults, flags
+
+ def com_fpdef(self, node):
+ # fpdef: NAME | '(' fplist ')'
+ if node[1][0] == token.LPAR:
+ return self.com_fplist(node[2])
+ return node[1][1]
+
+ def com_fplist(self, node):
+ # fplist: fpdef (',' fpdef)* [',']
+ if len(node) == 2:
+ return self.com_fpdef(node[1])
+ list = []
+ for i in range(1, len(node), 2):
+ list.append(self.com_fpdef(node[i]))
+ return tuple(list)
+
+ def com_dotted_name(self, node):
+ # String together the dotted names and return the string
+ name = ""
+ for n in node:
+ if type(n) == type(()) and n[0] == 1:
+ name = name + n[1] + '.'
+ return name[:-1]
+
+ def com_dotted_as_name(self, node):
+ assert node[0] == symbol.dotted_as_name
+ node = node[1:]
+ dot = self.com_dotted_name(node[0][1:])
+ if len(node) == 1:
+ return dot, None
+ assert node[1][1] == 'as'
+ assert node[2][0] == token.NAME
+ return dot, node[2][1]
+
+ def com_dotted_as_names(self, node):
+ assert node[0] == symbol.dotted_as_names
+ node = node[1:]
+ names = [self.com_dotted_as_name(node[0])]
+ for i in range(2, len(node), 2):
+ names.append(self.com_dotted_as_name(node[i]))
+ return names
+
+ def com_import_as_name(self, node):
+ assert node[0] == symbol.import_as_name
+ node = node[1:]
+ assert node[0][0] == token.NAME
+ if len(node) == 1:
+ return node[0][1], None
+ assert node[1][1] == 'as', node
+ assert node[2][0] == token.NAME
+ return node[0][1], node[2][1]
+
+ def com_import_as_names(self, node):
+ assert node[0] == symbol.import_as_names
+ node = node[1:]
+ names = [self.com_import_as_name(node[0])]
+ for i in range(2, len(node), 2):
+ names.append(self.com_import_as_name(node[i]))
+ return names
+
+ def com_bases(self, node):
+ bases = []
+ for i in range(1, len(node), 2):
+ bases.append(self.com_node(node[i]))
+ return bases
+
+ def com_try_except_finally(self, nodelist):
+ # ('try' ':' suite
+ # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite]
+ # | 'finally' ':' suite))
+
+ if nodelist[3][0] == token.NAME:
+ # first clause is a finally clause: only try-finally
+ return TryFinally(self.com_node(nodelist[2]),
+ self.com_node(nodelist[5]),
+ lineno=nodelist[0][2])
+
+ #tryexcept: [TryNode, [except_clauses], elseNode)]
+ clauses = []
+ elseNode = None
+ finallyNode = None
+ for i in range(3, len(nodelist), 3):
+ node = nodelist[i]
+ if node[0] == symbol.except_clause:
+ # except_clause: 'except' [expr [(',' | 'as') expr]] */
+ if len(node) > 2:
+ expr1 = self.com_node(node[2])
+ if len(node) > 4:
+ expr2 = self.com_assign(node[4], OP_ASSIGN)
+ else:
+ expr2 = None
+ else:
+ expr1 = expr2 = None
+ clauses.append((expr1, expr2, self.com_node(nodelist[i+2])))
+
+ if node[0] == token.NAME:
+ if node[1] == 'else':
+ elseNode = self.com_node(nodelist[i+2])
+ elif node[1] == 'finally':
+ finallyNode = self.com_node(nodelist[i+2])
+ try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode,
+ lineno=nodelist[0][2])
+ if finallyNode:
+ return TryFinally(try_except, finallyNode, lineno=nodelist[0][2])
+ else:
+ return try_except
+
+ def com_with(self, nodelist):
+ # with_stmt: 'with' with_item (',' with_item)* ':' suite
+ body = self.com_node(nodelist[-1])
+ for i in range(len(nodelist) - 3, 0, -2):
+ ret = self.com_with_item(nodelist[i], body, nodelist[0][2])
+ if i == 1:
+ return ret
+ body = ret
+
+ def com_with_item(self, nodelist, body, lineno):
+ # with_item: test ['as' expr]
+ if len(nodelist) == 4:
+ var = self.com_assign(nodelist[3], OP_ASSIGN)
+ else:
+ var = None
+ expr = self.com_node(nodelist[1])
+ return With(expr, var, body, lineno=lineno)
+
+ def com_augassign_op(self, node):
+ assert node[0] == symbol.augassign
+ return node[1]
+
+ def com_augassign(self, node):
+ """Return node suitable for lvalue of augmented assignment
+
+ Names, slices, and attributes are the only allowable nodes.
+ """
+ l = self.com_node(node)
+ if l.__class__ in (Name, Slice, Subscript, Getattr):
+ return l
+ raise SyntaxError, "can't assign to %s" % l.__class__.__name__
+
+ def com_assign(self, node, assigning):
+ # return a node suitable for use as an "lvalue"
+ # loop to avoid trivial recursion
+ while 1:
+ t = node[0]
+ if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp):
+ if len(node) > 2:
+ return self.com_assign_tuple(node, assigning)
+ node = node[1]
+ elif t in _assign_types:
+ if len(node) > 2:
+ raise SyntaxError, "can't assign to operator"
+ node = node[1]
+ elif t == symbol.power:
+ if node[1][0] != symbol.atom:
+ raise SyntaxError, "can't assign to operator"
+ if len(node) > 2:
+ primary = self.com_node(node[1])
+ for i in range(2, len(node)-1):
+ ch = node[i]
+ if ch[0] == token.DOUBLESTAR:
+ raise SyntaxError, "can't assign to operator"
+ primary = self.com_apply_trailer(primary, ch)
+ return self.com_assign_trailer(primary, node[-1],
+ assigning)
+ node = node[1]
+ elif t == symbol.atom:
+ t = node[1][0]
+ if t == token.LPAR:
+ node = node[2]
+ if node[0] == token.RPAR:
+ raise SyntaxError, "can't assign to ()"
+ elif t == token.LSQB:
+ node = node[2]
+ if node[0] == token.RSQB:
+ raise SyntaxError, "can't assign to []"
+ return self.com_assign_list(node, assigning)
+ elif t == token.NAME:
+ return self.com_assign_name(node[1], assigning)
+ else:
+ raise SyntaxError, "can't assign to literal"
+ else:
+ raise SyntaxError, "bad assignment (%s)" % t
+
+ def com_assign_tuple(self, node, assigning):
+ assigns = []
+ for i in range(1, len(node), 2):
+ assigns.append(self.com_assign(node[i], assigning))
+ return AssTuple(assigns, lineno=extractLineNo(node))
+
+ def com_assign_list(self, node, assigning):
+ assigns = []
+ for i in range(1, len(node), 2):
+ if i + 1 < len(node):
+ if node[i + 1][0] == symbol.list_for:
+ raise SyntaxError, "can't assign to list comprehension"
+ assert node[i + 1][0] == token.COMMA, node[i + 1]
+ assigns.append(self.com_assign(node[i], assigning))
+ return AssList(assigns, lineno=extractLineNo(node))
+
+ def com_assign_name(self, node, assigning):
+ return AssName(node[1], assigning, lineno=node[2])
+
+ def com_assign_trailer(self, primary, node, assigning):
+ t = node[1][0]
+ if t == token.DOT:
+ return self.com_assign_attr(primary, node[2], assigning)
+ if t == token.LSQB:
+ return self.com_subscriptlist(primary, node[2], assigning)
+ if t == token.LPAR:
+ raise SyntaxError, "can't assign to function call"
+ raise SyntaxError, "unknown trailer type: %s" % t
+
+ def com_assign_attr(self, primary, node, assigning):
+ return AssAttr(primary, node[1], assigning, lineno=node[-1])
+
+ def com_binary(self, constructor, nodelist):
+ "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
+ l = len(nodelist)
+ if l == 1:
+ n = nodelist[0]
+ return self.lookup_node(n)(n[1:])
+ items = []
+ for i in range(0, l, 2):
+ n = nodelist[i]
+ items.append(self.lookup_node(n)(n[1:]))
+ return constructor(items, lineno=extractLineNo(nodelist))
+
+ def com_stmt(self, node):
+ result = self.lookup_node(node)(node[1:])
+ assert result is not None
+ if isinstance(result, Stmt):
+ return result
+ return Stmt([result])
+
+ def com_append_stmt(self, stmts, node):
+ result = self.lookup_node(node)(node[1:])
+ assert result is not None
+ if isinstance(result, Stmt):
+ stmts.extend(result.nodes)
+ else:
+ stmts.append(result)
+
+ def com_list_constructor(self, nodelist):
+ # listmaker: test ( list_for | (',' test)* [','] )
+ values = []
+ for i in range(1, len(nodelist)):
+ if nodelist[i][0] == symbol.list_for:
+ assert len(nodelist[i:]) == 1
+ return self.com_list_comprehension(values[0],
+ nodelist[i])
+ elif nodelist[i][0] == token.COMMA:
+ continue
+ values.append(self.com_node(nodelist[i]))
+ return List(values, lineno=values[0].lineno)
+
+ def com_list_comprehension(self, expr, node):
+ return self.com_comprehension(expr, None, node, 'list')
+
+ def com_comprehension(self, expr1, expr2, node, type):
+ # list_iter: list_for | list_if
+ # list_for: 'for' exprlist 'in' testlist [list_iter]
+ # list_if: 'if' test [list_iter]
+
+ # XXX should raise SyntaxError for assignment
+ # XXX(avassalotti) Set and dict comprehensions should have generator
+ # semantics. In other words, they shouldn't leak
+ # variables outside of the comprehension's scope.
+
+ lineno = node[1][2]
+ fors = []
+ while node:
+ t = node[1][1]
+ if t == 'for':
+ assignNode = self.com_assign(node[2], OP_ASSIGN)
+ compNode = self.com_node(node[4])
+ newfor = ListCompFor(assignNode, compNode, [])
+ newfor.lineno = node[1][2]
+ fors.append(newfor)
+ if len(node) == 5:
+ node = None
+ elif type == 'list':
+ node = self.com_list_iter(node[5])
+ else:
+ node = self.com_comp_iter(node[5])
+ elif t == 'if':
+ test = self.com_node(node[2])
+ newif = ListCompIf(test, lineno=node[1][2])
+ newfor.ifs.append(newif)
+ if len(node) == 3:
+ node = None
+ elif type == 'list':
+ node = self.com_list_iter(node[3])
+ else:
+ node = self.com_comp_iter(node[3])
+ else:
+ raise SyntaxError, \
+ ("unexpected comprehension element: %s %d"
+ % (node, lineno))
+ if type == 'list':
+ return ListComp(expr1, fors, lineno=lineno)
+ elif type == 'set':
+ return SetComp(expr1, fors, lineno=lineno)
+ elif type == 'dict':
+ return DictComp(expr1, expr2, fors, lineno=lineno)
+ else:
+ raise ValueError("unexpected comprehension type: " + repr(type))
+
+ def com_list_iter(self, node):
+ assert node[0] == symbol.list_iter
+ return node[1]
+
+ def com_comp_iter(self, node):
+ assert node[0] == symbol.comp_iter
+ return node[1]
+
+ def com_generator_expression(self, expr, node):
+ # comp_iter: comp_for | comp_if
+ # comp_for: 'for' exprlist 'in' test [comp_iter]
+ # comp_if: 'if' test [comp_iter]
+
+ lineno = node[1][2]
+ fors = []
+ while node:
+ t = node[1][1]
+ if t == 'for':
+ assignNode = self.com_assign(node[2], OP_ASSIGN)
+ genNode = self.com_node(node[4])
+ newfor = GenExprFor(assignNode, genNode, [],
+ lineno=node[1][2])
+ fors.append(newfor)
+ if (len(node)) == 5:
+ node = None
+ else:
+ node = self.com_comp_iter(node[5])
+ elif t == 'if':
+ test = self.com_node(node[2])
+ newif = GenExprIf(test, lineno=node[1][2])
+ newfor.ifs.append(newif)
+ if len(node) == 3:
+ node = None
+ else:
+ node = self.com_comp_iter(node[3])
+ else:
+ raise SyntaxError, \
+ ("unexpected generator expression element: %s %d"
+ % (node, lineno))
+ fors[0].is_outmost = True
+ return GenExpr(GenExprInner(expr, fors), lineno=lineno)
+
+ def com_dictorsetmaker(self, nodelist):
+ # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
+ # (test (comp_for | (',' test)* [','])) )
+ assert nodelist[0] == symbol.dictorsetmaker
+ nodelist = nodelist[1:]
+ if len(nodelist) == 1 or nodelist[1][0] == token.COMMA:
+ # set literal
+ items = []
+ for i in range(0, len(nodelist), 2):
+ items.append(self.com_node(nodelist[i]))
+ return Set(items, lineno=items[0].lineno)
+ elif nodelist[1][0] == symbol.comp_for:
+ # set comprehension
+ expr = self.com_node(nodelist[0])
+ return self.com_comprehension(expr, None, nodelist[1], 'set')
+ elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for:
+ # dict comprehension
+ assert nodelist[1][0] == token.COLON
+ key = self.com_node(nodelist[0])
+ value = self.com_node(nodelist[2])
+ return self.com_comprehension(key, value, nodelist[3], 'dict')
+ else:
+ # dict literal
+ items = []
+ for i in range(0, len(nodelist), 4):
+ items.append((self.com_node(nodelist[i]),
+ self.com_node(nodelist[i+2])))
+ return Dict(items, lineno=items[0][0].lineno)
+
+ def com_apply_trailer(self, primaryNode, nodelist):
+ t = nodelist[1][0]
+ if t == token.LPAR:
+ return self.com_call_function(primaryNode, nodelist[2])
+ if t == token.DOT:
+ return self.com_select_member(primaryNode, nodelist[2])
+ if t == token.LSQB:
+ return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY)
+
+ raise SyntaxError, 'unknown node type: %s' % t
+
+ def com_select_member(self, primaryNode, nodelist):
+ if nodelist[0] != token.NAME:
+ raise SyntaxError, "member must be a name"
+ return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
+
+ def com_call_function(self, primaryNode, nodelist):
+ if nodelist[0] == token.RPAR:
+ return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
+ args = []
+ kw = 0
+ star_node = dstar_node = None
+ len_nodelist = len(nodelist)
+ i = 1
+ while i < len_nodelist:
+ node = nodelist[i]
+
+ if node[0]==token.STAR:
+ if star_node is not None:
+ raise SyntaxError, 'already have the varargs indentifier'
+ star_node = self.com_node(nodelist[i+1])
+ i = i + 3
+ continue
+ elif node[0]==token.DOUBLESTAR:
+ if dstar_node is not None:
+ raise SyntaxError, 'already have the kwargs indentifier'
+ dstar_node = self.com_node(nodelist[i+1])
+ i = i + 3
+ continue
+
+ # positional or named parameters
+ kw, result = self.com_argument(node, kw, star_node)
+
+ if len_nodelist != 2 and isinstance(result, GenExpr) \
+ and len(node) == 3 and node[2][0] == symbol.comp_for:
+ # allow f(x for x in y), but reject f(x for x in y, 1)
+ # should use f((x for x in y), 1) instead of f(x for x in y, 1)
+ raise SyntaxError, 'generator expression needs parenthesis'
+
+ args.append(result)
+ i = i + 2
+
+ return CallFunc(primaryNode, args, star_node, dstar_node,
+ lineno=extractLineNo(nodelist))
+
+ def com_argument(self, nodelist, kw, star_node):
+ if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for:
+ test = self.com_node(nodelist[1])
+ return 0, self.com_generator_expression(test, nodelist[2])
+ if len(nodelist) == 2:
+ if kw:
+ raise SyntaxError, "non-keyword arg after keyword arg"
+ if star_node:
+ raise SyntaxError, "only named arguments may follow *expression"
+ return 0, self.com_node(nodelist[1])
+ result = self.com_node(nodelist[3])
+ n = nodelist[1]
+ while len(n) == 2 and n[0] != token.NAME:
+ n = n[1]
+ if n[0] != token.NAME:
+ raise SyntaxError, "keyword can't be an expression (%s)"%n[0]
+ node = Keyword(n[1], result, lineno=n[2])
+ return 1, node
+
+ def com_subscriptlist(self, primary, nodelist, assigning):
+ # slicing: simple_slicing | extended_slicing
+ # simple_slicing: primary "[" short_slice "]"
+ # extended_slicing: primary "[" slice_list "]"
+ # slice_list: slice_item ("," slice_item)* [","]
+
+ # backwards compat slice for '[i:j]'
+ if len(nodelist) == 2:
+ sub = nodelist[1]
+ if (sub[1][0] == token.COLON or \
+ (len(sub) > 2 and sub[2][0] == token.COLON)) and \
+ sub[-1][0] != symbol.sliceop:
+ return self.com_slice(primary, sub, assigning)
+
+ subscripts = []
+ for i in range(1, len(nodelist), 2):
+ subscripts.append(self.com_subscript(nodelist[i]))
+ return Subscript(primary, assigning, subscripts,
+ lineno=extractLineNo(nodelist))
+
+ def com_subscript(self, node):
+ # slice_item: expression | proper_slice | ellipsis
+ ch = node[1]
+ t = ch[0]
+ if t == token.DOT and node[2][0] == token.DOT:
+ return Ellipsis()
+ if t == token.COLON or len(node) > 2:
+ return self.com_sliceobj(node)
+ return self.com_node(ch)
+
+ def com_sliceobj(self, node):
+ # proper_slice: short_slice | long_slice
+ # short_slice: [lower_bound] ":" [upper_bound]
+ # long_slice: short_slice ":" [stride]
+ # lower_bound: expression
+ # upper_bound: expression
+ # stride: expression
+ #
+ # Note: a stride may be further slicing...
+
+ items = []
+
+ if node[1][0] == token.COLON:
+ items.append(Const(None))
+ i = 2
+ else:
+ items.append(self.com_node(node[1]))
+ # i == 2 is a COLON
+ i = 3
+
+ if i < len(node) and node[i][0] == symbol.test:
+ items.append(self.com_node(node[i]))
+ i = i + 1
+ else:
+ items.append(Const(None))
+
+ # a short_slice has been built. look for long_slice now by looking
+ # for strides...
+ for j in range(i, len(node)):
+ ch = node[j]
+ if len(ch) == 2:
+ items.append(Const(None))
+ else:
+ items.append(self.com_node(ch[2]))
+ return Sliceobj(items, lineno=extractLineNo(node))
+
+ def com_slice(self, primary, node, assigning):
+ # short_slice: [lower_bound] ":" [upper_bound]
+ lower = upper = None
+ if len(node) == 3:
+ if node[1][0] == token.COLON:
+ upper = self.com_node(node[2])
+ else:
+ lower = self.com_node(node[1])
+ elif len(node) == 4:
+ lower = self.com_node(node[1])
+ upper = self.com_node(node[3])
+ return Slice(primary, assigning, lower, upper,
+ lineno=extractLineNo(node))
+
+ def get_docstring(self, node, n=None):
+ if n is None:
+ n = node[0]
+ node = node[1:]
+ if n == symbol.suite:
+ if len(node) == 1:
+ return self.get_docstring(node[0])
+ for sub in node:
+ if sub[0] == symbol.stmt:
+ return self.get_docstring(sub)
+ return None
+ if n == symbol.file_input:
+ for sub in node:
+ if sub[0] == symbol.stmt:
+ return self.get_docstring(sub)
+ return None
+ if n == symbol.atom:
+ if node[0][0] == token.STRING:
+ s = ''
+ for t in node:
+ s = s + eval(t[1])
+ return s
+ return None
+ if n == symbol.stmt or n == symbol.simple_stmt \
+ or n == symbol.small_stmt:
+ return self.get_docstring(node[0])
+ if n in _doc_nodes and len(node) == 1:
+ return self.get_docstring(node[0])
+ return None
+
+
+_doc_nodes = [
+ symbol.expr_stmt,
+ symbol.testlist,
+ symbol.testlist_safe,
+ symbol.test,
+ symbol.or_test,
+ symbol.and_test,
+ symbol.not_test,
+ symbol.comparison,
+ symbol.expr,
+ symbol.xor_expr,
+ symbol.and_expr,
+ symbol.shift_expr,
+ symbol.arith_expr,
+ symbol.term,
+ symbol.factor,
+ symbol.power,
+ ]
+
+# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
+# | 'in' | 'not' 'in' | 'is' | 'is' 'not'
+_cmp_types = {
+ token.LESS : '<',
+ token.GREATER : '>',
+ token.EQEQUAL : '==',
+ token.EQUAL : '==',
+ token.LESSEQUAL : '<=',
+ token.GREATEREQUAL : '>=',
+ token.NOTEQUAL : '!=',
+ }
+
+_legal_node_types = [
+ symbol.funcdef,
+ symbol.classdef,
+ symbol.stmt,
+ symbol.small_stmt,
+ symbol.flow_stmt,
+ symbol.simple_stmt,
+ symbol.compound_stmt,
+ symbol.expr_stmt,
+ symbol.print_stmt,
+ symbol.del_stmt,
+ symbol.pass_stmt,
+ symbol.break_stmt,
+ symbol.continue_stmt,
+ symbol.return_stmt,
+ symbol.raise_stmt,
+ symbol.import_stmt,
+ symbol.global_stmt,
+ symbol.exec_stmt,
+ symbol.assert_stmt,
+ symbol.if_stmt,
+ symbol.while_stmt,
+ symbol.for_stmt,
+ symbol.try_stmt,
+ symbol.with_stmt,
+ symbol.suite,
+ symbol.testlist,
+ symbol.testlist_safe,
+ symbol.test,
+ symbol.and_test,
+ symbol.not_test,
+ symbol.comparison,
+ symbol.exprlist,
+ symbol.expr,
+ symbol.xor_expr,
+ symbol.and_expr,
+ symbol.shift_expr,
+ symbol.arith_expr,
+ symbol.term,
+ symbol.factor,
+ symbol.power,
+ symbol.atom,
+ ]
+
+if hasattr(symbol, 'yield_stmt'):
+ _legal_node_types.append(symbol.yield_stmt)
+if hasattr(symbol, 'yield_expr'):
+ _legal_node_types.append(symbol.yield_expr)
+
+_assign_types = [
+ symbol.test,
+ symbol.or_test,
+ symbol.and_test,
+ symbol.not_test,
+ symbol.comparison,
+ symbol.expr,
+ symbol.xor_expr,
+ symbol.and_expr,
+ symbol.shift_expr,
+ symbol.arith_expr,
+ symbol.term,
+ symbol.factor,
+ ]
+
+_names = {}
+for k, v in symbol.sym_name.items():
+ _names[k] = v
+for k, v in token.tok_name.items():
+ _names[k] = v
+
+def debug_tree(tree):
+ l = []
+ for elt in tree:
+ if isinstance(elt, int):
+ l.append(_names.get(elt, elt))
+ elif isinstance(elt, str):
+ l.append(elt)
+ else:
+ l.append(debug_tree(elt))
+ return l
diff --git a/lib/python2.7/compiler/visitor.py b/lib/python2.7/compiler/visitor.py
new file mode 100644
index 0000000..f10f560
--- /dev/null
+++ b/lib/python2.7/compiler/visitor.py
@@ -0,0 +1,113 @@
+from compiler import ast
+
+# XXX should probably rename ASTVisitor to ASTWalker
+# XXX can it be made even more generic?
+
+class ASTVisitor:
+ """Performs a depth-first walk of the AST
+
+ The ASTVisitor will walk the AST, performing either a preorder or
+ postorder traversal depending on which method is called.
+
+ methods:
+ preorder(tree, visitor)
+ postorder(tree, visitor)
+ tree: an instance of ast.Node
+ visitor: an instance with visitXXX methods
+
+ The ASTVisitor is responsible for walking over the tree in the
+ correct order. For each node, it checks the visitor argument for
+ a method named 'visitNodeType' where NodeType is the name of the
+ node's class, e.g. Class. If the method exists, it is called
+ with the node as its sole argument.
+
+ The visitor method for a particular node type can control how
+ child nodes are visited during a preorder walk. (It can't control
+ the order during a postorder walk, because it is called _after_
+ the walk has occurred.) The ASTVisitor modifies the visitor
+ argument by adding a visit method to the visitor; this method can
+ be used to visit a child node of arbitrary type.
+ """
+
+ VERBOSE = 0
+
+ def __init__(self):
+ self.node = None
+ self._cache = {}
+
+ def default(self, node, *args):
+ for child in node.getChildNodes():
+ self.dispatch(child, *args)
+
+ def dispatch(self, node, *args):
+ self.node = node
+ klass = node.__class__
+ meth = self._cache.get(klass, None)
+ if meth is None:
+ className = klass.__name__
+ meth = getattr(self.visitor, 'visit' + className, self.default)
+ self._cache[klass] = meth
+## if self.VERBOSE > 0:
+## className = klass.__name__
+## if self.VERBOSE == 1:
+## if meth == 0:
+## print "dispatch", className
+## else:
+## print "dispatch", className, (meth and meth.__name__ or '')
+ return meth(node, *args)
+
+ def preorder(self, tree, visitor, *args):
+ """Do preorder walk of tree using visitor"""
+ self.visitor = visitor
+ visitor.visit = self.dispatch
+ self.dispatch(tree, *args) # XXX *args make sense?
+
+class ExampleASTVisitor(ASTVisitor):
+ """Prints examples of the nodes that aren't visited
+
+ This visitor-driver is only useful for development, when it's
+ helpful to develop a visitor incrementally, and get feedback on what
+ you still have to do.
+ """
+ examples = {}
+
+ def dispatch(self, node, *args):
+ self.node = node
+ meth = self._cache.get(node.__class__, None)
+ className = node.__class__.__name__
+ if meth is None:
+ meth = getattr(self.visitor, 'visit' + className, 0)
+ self._cache[node.__class__] = meth
+ if self.VERBOSE > 1:
+ print "dispatch", className, (meth and meth.__name__ or '')
+ if meth:
+ meth(node, *args)
+ elif self.VERBOSE > 0:
+ klass = node.__class__
+ if klass not in self.examples:
+ self.examples[klass] = klass
+ print
+ print self.visitor
+ print klass
+ for attr in dir(node):
+ if attr[0] != '_':
+ print "\t", "%-12.12s" % attr, getattr(node, attr)
+ print
+ return self.default(node, *args)
+
+# XXX this is an API change
+
+_walker = ASTVisitor
+def walk(tree, visitor, walker=None, verbose=None):
+ if walker is None:
+ walker = _walker()
+ if verbose is not None:
+ walker.VERBOSE = verbose
+ walker.preorder(tree, visitor)
+ return walker.visitor
+
+def dumpNode(node):
+ print node.__class__
+ for attr in dir(node):
+ if attr[0] != '_':
+ print "\t", "%-10.10s" % attr, getattr(node, attr)