changeset 13:38b0d63697b1

Import the full AST grammar from CPython 3.5.
author Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
date Thu, 02 Jun 2016 05:33:23 +0100
parents 0e96c5bc401d
children 719a27f1c1c7
files src/ast_convert.rs src/ast_dump.rs src/ast_scope.rs src/main.rs src/python_ast.rs
diffstat 5 files changed, 612 insertions(+), 279 deletions(-) [+]
line wrap: on
line diff
--- a/src/ast_convert.rs
+++ b/src/ast_convert.rs
@@ -1,4 +1,4 @@
-use python_ast::{Module, Statement, Expr, BinOp, UnaryOp};
+use python_ast::{Module, stmt, expr, expr_context, cmpop, operator, unaryop, arguments, arg, alias};
 
 use cpython::{Python, PyObject};
 use cpython::ObjectProtocol; //for call method
@@ -30,7 +30,7 @@ fn parse_expr_vec(py: Python, ast: PyObj
             let arg = arg.unwrap();
             let arg = parse_expr(py, arg);
             arguments.push(match arg {
-                Expr::Name(arg) => arg,
+                expr::Name(arg, expr_context::Load) => arg,
                 _ => panic!()
             });
         }
@@ -40,7 +40,7 @@ fn parse_expr_vec(py: Python, ast: PyObj
     }
 }
 
-fn parse_unaryop(py: Python, ast: PyObject) -> UnaryOp {
+fn parse_unaryop(py: Python, ast: PyObject) -> unaryop {
     let builtins_module = py.import("builtins").unwrap();
     let isinstance = builtins_module.get(py, "isinstance").unwrap();
 
@@ -50,22 +50,57 @@ fn parse_unaryop(py: Python, ast: PyObje
 
     let ast_module = py.import("ast").unwrap();
     let ast_type = ast_module.get(py, "AST").unwrap();
+    let invert_type = ast_module.get(py, "Invert").unwrap();
+    let not_type = ast_module.get(py, "Not").unwrap();
     let uadd_type = ast_module.get(py, "UAdd").unwrap();
     let usub_type = ast_module.get(py, "USub").unwrap();
 
     assert!(is_instance(&ast, &ast_type));
 
-    if is_instance(&ast, &uadd_type) {
-        UnaryOp::UAdd
+    if is_instance(&ast, &invert_type) {
+        unaryop::Invert
+    } else if is_instance(&ast, &not_type) {
+        unaryop::Not
+    } else if is_instance(&ast, &uadd_type) {
+        unaryop::UAdd
     } else if is_instance(&ast, &usub_type) {
-        UnaryOp::USub
+        unaryop::USub
     } else {
-        println!("UnaryOp {}", ast);
-        UnaryOp::Error
+        unreachable!()
     }
 }
 
-fn parse_binop(py: Python, ast: PyObject) -> BinOp {
+fn parse_cmpop(py: Python, ast: PyObject) -> cmpop {
+    let builtins_module = py.import("builtins").unwrap();
+    let isinstance = builtins_module.get(py, "isinstance").unwrap();
+
+    let is_instance = |object: &PyObject, type_: &PyObject| {
+        return isinstance.call(py, (object, type_), None).unwrap().is_true(py).unwrap();
+    };
+
+    let ast_module = py.import("ast").unwrap();
+    let ast_type = ast_module.get(py, "AST").unwrap();
+    let eq_type = ast_module.get(py, "Eq").unwrap();
+    let noteq_type = ast_module.get(py, "NotEq").unwrap();
+    let lt_type = ast_module.get(py, "Lt").unwrap();
+    let gt_type = ast_module.get(py, "Gt").unwrap();
+
+    assert!(is_instance(&ast, &ast_type));
+
+    if is_instance(&ast, &eq_type) {
+        cmpop::Eq
+    } else if is_instance(&ast, &noteq_type) {
+        cmpop::NotEq
+    } else if is_instance(&ast, &lt_type) {
+        cmpop::Lt
+    } else if is_instance(&ast, &gt_type) {
+        cmpop::Gt
+    } else {
+        unreachable!()
+    }
+}
+
+fn parse_operator(py: Python, ast: PyObject) -> operator {
     let builtins_module = py.import("builtins").unwrap();
     let isinstance = builtins_module.get(py, "isinstance").unwrap();
 
@@ -76,36 +111,54 @@ fn parse_binop(py: Python, ast: PyObject
     let ast_module = py.import("ast").unwrap();
     let ast_type = ast_module.get(py, "AST").unwrap();
     let add_type = ast_module.get(py, "Add").unwrap();
+    let sub_type = ast_module.get(py, "Sub").unwrap();
     let mult_type = ast_module.get(py, "Mult").unwrap();
-    let eq_type = ast_module.get(py, "Eq").unwrap();
-    let lt_type = ast_module.get(py, "Lt").unwrap();
-    let gt_type = ast_module.get(py, "Gt").unwrap();
-    let sub_type = ast_module.get(py, "Sub").unwrap();
+    let matmult_type = ast_module.get(py, "MatMult").unwrap();
     let div_type = ast_module.get(py, "Div").unwrap();
+    let mod_type = ast_module.get(py, "Mod").unwrap();
+    let pow_type = ast_module.get(py, "Pow").unwrap();
+    let lshift_type = ast_module.get(py, "LShift").unwrap();
+    let rshift_type = ast_module.get(py, "RShift").unwrap();
+    let bitor_type = ast_module.get(py, "BitOr").unwrap();
+    let bitxor_type = ast_module.get(py, "BitXor").unwrap();
+    let bitand_type = ast_module.get(py, "BitAnd").unwrap();
+    let floordiv_type = ast_module.get(py, "FloorDiv").unwrap();
 
     assert!(is_instance(&ast, &ast_type));
 
     if is_instance(&ast, &add_type) {
-        BinOp::BinAdd
+        operator::Add
+    } else if is_instance(&ast, &sub_type) {
+        operator::Sub
     } else if is_instance(&ast, &mult_type) {
-        BinOp::BinMult
-    } else if is_instance(&ast, &eq_type) {
-        BinOp::BinEq
-    } else if is_instance(&ast, &lt_type) {
-        BinOp::BinLt
-    } else if is_instance(&ast, &gt_type) {
-        BinOp::BinGt
-    } else if is_instance(&ast, &sub_type) {
-        BinOp::Sub
+        operator::Mult
+    } else if is_instance(&ast, &matmult_type) {
+        operator::MatMult
     } else if is_instance(&ast, &div_type) {
-        BinOp::Div
+        operator::Div
+    } else if is_instance(&ast, &mod_type) {
+        operator::Mod
+    } else if is_instance(&ast, &pow_type) {
+        operator::Pow
+    } else if is_instance(&ast, &lshift_type) {
+        operator::LShift
+    } else if is_instance(&ast, &rshift_type) {
+        operator::RShift
+    } else if is_instance(&ast, &bitor_type) {
+        operator::BitOr
+    } else if is_instance(&ast, &bitxor_type) {
+        operator::BitXor
+    } else if is_instance(&ast, &bitand_type) {
+        operator::BitAnd
+    } else if is_instance(&ast, &floordiv_type) {
+        operator::FloorDiv
     } else {
-        println!("BinOp {}", ast);
-        BinOp::Error
+        println!("operator {}", ast);
+        panic!()
     }
 }
 
-fn parse_expr(py: Python, ast: PyObject) -> Expr {
+fn parse_expr(py: Python, ast: PyObject) -> expr {
     let builtins_module = py.import("builtins").unwrap();
     let isinstance = builtins_module.get(py, "isinstance").unwrap();
 
@@ -126,14 +179,13 @@ fn parse_expr(py: Python, ast: PyObject)
     let list_type = ast_module.get(py, "List").unwrap();
     let compare_type = ast_module.get(py, "Compare").unwrap();
     let call_type = ast_module.get(py, "Call").unwrap();
-    let alias_type = ast_module.get(py, "alias").unwrap();
 
     assert!(is_instance(&ast, &ast_type));
 
     if is_instance(&ast, &arg_type) {
         let arg = ast.getattr(py, "arg").unwrap();
         let arg = get_str(py, arg);
-        Expr::Name(arg)
+        expr::Name(arg, expr_context::Load)
     } else if is_instance(&ast, &attribute_type) {
         let value = ast.getattr(py, "value").unwrap();
         let attr = ast.getattr(py, "attr").unwrap();
@@ -141,23 +193,23 @@ fn parse_expr(py: Python, ast: PyObject)
         let value = parse_expr(py, value);
         let attr = get_str(py, attr);
 
-        Expr::Attribute(Box::new(value), attr)
+        expr::Attribute(Box::new(value), attr, expr_context::Load)
     } else if is_instance(&ast, &name_type) {
         let id = ast.getattr(py, "id").unwrap();
         let id = get_str(py, id);
-        Expr::Name(id)
+        expr::Name(id, expr_context::Load)
     } else if is_instance(&ast, &name_constant_type) {
         let value = ast.getattr(py, "value").unwrap();
         let value = get_str(py, value);
-        Expr::NameConstant(value)
+        expr::NameConstant(value)
     } else if is_instance(&ast, &num_type) {
         let n = ast.getattr(py, "n").unwrap();
         let n = get_str(py, n);
-        Expr::Num(n)
+        expr::Num(n)
     } else if is_instance(&ast, &str_type) {
         let s = ast.getattr(py, "s").unwrap();
         let s = get_str(py, s);
-        Expr::Str(s)
+        expr::Str(s)
     } else if is_instance(&ast, &list_type) {
         let elts = ast.getattr(py, "elts").unwrap();
 
@@ -167,7 +219,7 @@ fn parse_expr(py: Python, ast: PyObject)
             elements.push(parse_expr(py, elt));
         }
 
-        Expr::List(elements)
+        expr::List(elements, expr_context::Load)
     } else if is_instance(&ast, &unary_op_type) {
         let op = ast.getattr(py, "op").unwrap();
         let operand = ast.getattr(py, "operand").unwrap();
@@ -175,21 +227,21 @@ fn parse_expr(py: Python, ast: PyObject)
         let op = parse_unaryop(py, op);
         let operand = parse_expr(py, operand);
 
-        Expr::UnaryOp(op, Box::new(operand))
+        expr::UnaryOp(op, Box::new(operand))
     } else if is_instance(&ast, &bin_op_type) {
         let left = ast.getattr(py, "left").unwrap();
         let op = ast.getattr(py, "op").unwrap();
         let right = ast.getattr(py, "right").unwrap();
 
         let left = parse_expr(py, left);
-        let op = parse_binop(py, op);
+        let op = parse_operator(py, op);
         let right = parse_expr(py, right);
 
-        Expr::BinOp(Box::new(left), op, Box::new(right))
+        expr::BinOp(Box::new(left), op, Box::new(right))
     } else if is_instance(&ast, &call_type) {
         let func = ast.getattr(py, "func").unwrap();
         let args = ast.getattr(py, "args").unwrap();
-        //let keywords = ast.getattr(py, "keywords").unwrap();
+        let keywords = ast.getattr(py, "keywords").unwrap();
 
         let func = parse_expr(py, func);
 
@@ -199,19 +251,15 @@ fn parse_expr(py: Python, ast: PyObject)
             arguments.push(parse_expr(py, arg));
         }
 
-        Expr::Call(Box::new(func), arguments)
-    } else if is_instance(&ast, &alias_type) {
-        let name = ast.getattr(py, "name").unwrap();
-        let asname = ast.getattr(py, "asname").unwrap();
+        /*
+        let mut kwargs = vec!();
+        for arg in keywords.iter(py).unwrap() {
+            let arg = arg.unwrap();
+            kwargs.push(parse_expr(py, arg));
+        }
+        */
 
-        let name = get_str(py, name);
-        let asname = if asname == py.None() {
-            None
-        } else {
-            Some(get_str(py, asname))
-        };
-
-        Expr::Alias(name, asname)
+        expr::Call(Box::new(func), arguments, vec!())
     } else if is_instance(&ast, &compare_type) {
         let left = ast.getattr(py, "left").unwrap();
         let ops = ast.getattr(py, "ops").unwrap();
@@ -224,7 +272,7 @@ fn parse_expr(py: Python, ast: PyObject)
         let mut new_ops = vec!();
         for op in ops {
             let op = op.unwrap();
-            let op = parse_binop(py, op);
+            let op = parse_cmpop(py, op);
             new_ops.push(op);
         }
 
@@ -235,16 +283,63 @@ fn parse_expr(py: Python, ast: PyObject)
             new_comparators.push(comparator);
         }
 
-        Expr::Compare(Box::new(left), new_ops, new_comparators)
+        expr::Compare(Box::new(left), new_ops, new_comparators)
     } else {
-        println!("Expr {}", ast);
-        Expr::Error
+        println!("expr {}", ast);
+        unreachable!()
     }
 }
 
-fn parse_statement(py: Python, ast: PyObject) -> Statement {
-    //Statement::FunctionDef(Expr::Name("function".to_string()), vec!(Expr::Name("a".to_string()), Expr::Name("b".to_string())), vec!())
-    //Statement::If(Expr::BinOp(BinOp::BinEq, Box::new(Expr::Name("__name__".to_string())), Box::new(Expr::Str("__main__".to_string()))), vec!(Statement::Expr(Expr::Call(Box::new(Expr::Name("function".to_string())), vec!(Expr::Num(1), Expr::Num(2))))))
+fn parse_arguments(py: Python, ast: PyObject) -> arguments {
+    let builtins_module = py.import("builtins").unwrap();
+    let isinstance = builtins_module.get(py, "isinstance").unwrap();
+
+    let is_instance = |object: &PyObject, type_: &PyObject| {
+        return isinstance.call(py, (object, type_), None).unwrap().is_true(py).unwrap();
+    };
+
+    let ast_module = py.import("ast").unwrap();
+    let ast_type = ast_module.get(py, "AST").unwrap();
+    let arguments_type = ast_module.get(py, "arguments").unwrap();
+    let arg_type = ast_module.get(py, "arg").unwrap();
+
+    assert!(is_instance(&ast, &ast_type));
+
+    if is_instance(&ast, &arguments_type) {
+        let args = arguments{
+            //args: Vec<arg>,
+            args: {
+                let args = ast.getattr(py, "args").unwrap();
+                let mut arguments = vec!();
+                for arg in args.iter(py).unwrap() {
+                    let arg = arg.unwrap();
+                    assert!(is_instance(&arg, &arg_type));
+                    let arg = get_str(py, arg);
+                    arguments.push(arg{arg: arg, annotation: None});
+                }
+                arguments
+            },
+            //vararg: Option<arg>,
+            vararg: None,
+            //kwonlyargs: Vec<arg>,
+            kwonlyargs: vec!(),
+            //kw_defaults: Vec<expr>,
+            kw_defaults: vec!(),
+            //kwarg: Option<arg>,
+            kwarg: None,
+            //defaults: Vec<expr>
+            defaults: vec!()
+        };
+        args
+    } else {
+        println!("arguments {}", ast);
+        panic!()
+    }
+}
+
+fn parse_statement(py: Python, ast: PyObject) -> stmt {
+    //stmt::FunctionDef(expr::Name("function".to_string()), vec!(expr::Name("a".to_string()), expr::Name("b".to_string())), vec!())
+    //stmt::If(expr::BinOp(BinOp::BinEq, Box::new(expr::Name("__name__".to_string())), Box::new(expr::Str("__main__".to_string()))), vec!(stmt::Expr(expr::Call(Box::new(expr::Name("function".to_string())), vec!(expr::Num(1), expr::Num(2))))))
 
     let builtins_module = py.import("builtins").unwrap();
     let isinstance = builtins_module.get(py, "isinstance").unwrap();
@@ -307,21 +402,14 @@ fn parse_statement(py: Python, ast: PyOb
             statements.push(statement);
         }
 
-        Statement::ClassDef(name, nodes, statements)
+        stmt::ClassDef(name, nodes, vec!(), statements, vec!())
     } else if is_instance(&ast, &function_def_type) {
         let name = ast.getattr(py, "name").unwrap();
         let args = ast.getattr(py, "args").unwrap();
         let body = ast.getattr(py, "body").unwrap();
 
         let name = get_str(py, name);
-        let args = parse_expr_vec(py, args);
-        /*
-        let mut arguments = vec!();
-        for arg in args.iter(py).unwrap() {
-            let arg = parse_expr(py, arg.unwrap());
-            arguments.push(arg);
-        }
-        */
+        let args = parse_arguments(py, args);
 
         let mut statements = vec!();
         for statement in body.iter(py).unwrap() {
@@ -329,7 +417,10 @@ fn parse_statement(py: Python, ast: PyOb
             statements.push(statement);
         }
 
-        Statement::FunctionDef(name, args, statements)
+        let decorators = vec!();
+        let returns = None;
+
+        stmt::FunctionDef(name, args, statements, decorators, returns)
     } else if is_instance(&ast, &global_type) {
         let names = ast.getattr(py, "names").unwrap();
 
@@ -340,7 +431,7 @@ fn parse_statement(py: Python, ast: PyOb
             globals.push(name);
         }
 
-        Statement::Global(globals)
+        stmt::Global(globals)
     } else if is_instance(&ast, &if_type) {
         let test = ast.getattr(py, "test").unwrap();
         let body = ast.getattr(py, "body").unwrap();
@@ -362,7 +453,7 @@ fn parse_statement(py: Python, ast: PyOb
             orelse_.push(statement);
         }
 
-        Statement::If(test, statements, orelse_)
+        stmt::If(test, statements, orelse_)
     } else if is_instance(&ast, &while_type) {
         let test = ast.getattr(py, "test").unwrap();
         let body = ast.getattr(py, "body").unwrap();
@@ -384,7 +475,7 @@ fn parse_statement(py: Python, ast: PyOb
             orelse_.push(statement);
         }
 
-        Statement::While(test, statements, orelse_)
+        stmt::While(test, statements, orelse_)
     } else if is_instance(&ast, &for_type) {
         let target = ast.getattr(py, "target").unwrap();
         let iter = ast.getattr(py, "iter").unwrap();
@@ -408,7 +499,7 @@ fn parse_statement(py: Python, ast: PyOb
             orelse_.push(statement);
         }
 
-        Statement::For(target, iter, statements, orelse_)
+        stmt::For(target, iter, statements, orelse_)
     } else if is_instance(&ast, &assign_type) {
         let targets = ast.getattr(py, "targets").unwrap();
         let value = ast.getattr(py, "value").unwrap();
@@ -422,21 +513,21 @@ fn parse_statement(py: Python, ast: PyOb
 
         let value = parse_expr(py, value);
 
-        Statement::Assign(arguments, value)
+        stmt::Assign(arguments, value)
     } else if is_instance(&ast, &aug_assign_type) {
         let target = ast.getattr(py, "target").unwrap();
         let op = ast.getattr(py, "op").unwrap();
         let value = ast.getattr(py, "value").unwrap();
 
         let target = parse_expr(py, target);
-        let op = parse_binop(py, op);
+        let op = parse_operator(py, op);
         let value = parse_expr(py, value);
 
-        Statement::AugAssign(target, op, value)
+        stmt::AugAssign(target, op, value)
     } else if is_instance(&ast, &import_from_type) {
         let module = ast.getattr(py, "module").unwrap();
         let names = ast.getattr(py, "names").unwrap();
-        //let level = ast.getattr(py, "level").unwrap();
+        let level = ast.getattr(py, "level").unwrap();
 
         let module = get_str(py, module);
 
@@ -444,23 +535,29 @@ fn parse_statement(py: Python, ast: PyOb
         for alias in names.iter(py).unwrap() {
             let alias = alias.unwrap();
             let alias = parse_expr(py, alias);
-            names_.push(alias);
+            println!("{:?}", alias);
+            // XXX
+            //names_.push(alias{name: alias, asname: alias});
         }
 
-        Statement::ImportFrom(module, names_)
+        stmt::ImportFrom(module, names_, None)
     } else if is_instance(&ast, &return_type) {
         let value = ast.getattr(py, "value").unwrap();
-        let value = parse_expr(py, value);
-        Statement::Return(value)
+        if value == py.None() {
+            stmt::Return(None)
+        } else {
+            let value = parse_expr(py, value);
+            stmt::Return(Some(value))
+        }
     } else if is_instance(&ast, &expr_type) {
         let value = ast.getattr(py, "value").unwrap();
         let value = parse_expr(py, value);
-        Statement::Expr(value)
+        stmt::Expr(value)
     } else if is_instance(&ast, &break_type) {
-        Statement::Break
+        stmt::Break
     } else {
-        println!("Statement {}", ast);
-        Statement::Error
+        println!("stmt {}", ast);
+        panic!()
     }
 }
 
--- a/src/ast_dump.rs
+++ b/src/ast_dump.rs
@@ -1,38 +1,116 @@
-use python_ast::{Module, Statement, Expr, BinOp, UnaryOp};
+use python_ast::{Module, stmt, expr, boolop, operator, unaryop, cmpop, arguments, arg, keyword};
 
 use std::iter;
 
-impl UnaryOp {
+impl boolop {
+    fn to_string(&self) -> &'static str {
+        match *self {
+            boolop::And => "and",
+            boolop::Or => "or"
+        }
+    }
+}
+
+impl operator {
     fn to_string(&self) -> &'static str {
         match *self {
-            UnaryOp::UAdd => "+",
-            UnaryOp::USub => "-",
-            UnaryOp::Error => "BinOp::Error"
+            operator::Add => "+",
+            operator::Sub => "-",
+            operator::Mult => "*",
+            operator::MatMult => "@",
+            operator::Div => "/",
+            operator::Mod => "%",
+            operator::Pow => "**",
+            operator::LShift => "<<",
+            operator::RShift => ">>",
+            operator::BitOr => "|",
+            operator::BitXor => "^",
+            operator::BitAnd => "&",
+            operator::FloorDiv => "//",
+        }
+    }
+}
+
+impl unaryop {
+    fn to_string(&self) -> &'static str {
+        match *self {
+            unaryop::Invert => "~",
+            unaryop::Not => "!",
+            unaryop::UAdd => "+",
+            unaryop::USub => "-"
         }
     }
 }
 
-impl BinOp {
+impl cmpop {
     fn to_string(&self) -> &'static str {
         match *self {
-            BinOp::BinAdd => "+",
-            BinOp::BinMult => "*",
-            BinOp::BinEq => "==",
-            BinOp::BinLt => "<",
-            BinOp::BinGt => ">",
-            BinOp::Sub => "-",
-            BinOp::Div => "/",
-            BinOp::Error => "BinOp::Error"
+            cmpop::Eq => "==",
+            cmpop::NotEq => "!=",
+            cmpop::Lt => "<",
+            cmpop::LtE => "<=",
+            cmpop::Gt => ">",
+            cmpop::GtE => ">=",
+            cmpop::Is => "is",
+            cmpop::IsNot => "is not",
+            cmpop::In => "in",
+            cmpop::NotIn => "not in",
         }
     }
 }
 
-impl Expr {
+fn args_to_string(args: Vec<expr>) -> String {
+    let mut arguments = vec!();
+    for arg in args {
+        arguments.push(arg.to_string());
+    }
+    arguments.join(", ")
+}
+
+fn kwargs_to_string(kwargs: Vec<keyword>) -> String {
+    let mut arguments = vec!();
+    for arg in kwargs {
+        match arg.arg {
+            Some(arg) => arguments.push(arg),
+            None => arguments.push("**kwarg".to_string())
+        }
+    }
+    arguments.join(", ")
+}
+
+fn arguments_to_string(args: arguments) -> String {
+    let mut arguments = vec!();
+    for arg in args.args {
+        arguments.push(arg.arg);
+    }
+    arguments.join(", ")
+}
+
+fn statements_to_string(indent: usize, body: Vec<stmt>) -> String {
+    let mut statements = vec!();
+    for statement in body {
+        statements.push(statement.to_string(indent + 1));
+    }
+    statements.join("\n")
+}
+
+fn if_else_statements_to_string(indent: usize, body: Vec<stmt>, orelse: Vec<stmt>) -> String {
+    let body = statements_to_string(indent, body);
+    if orelse.is_empty() {
+        body
+    } else {
+        format!("{}\n{}else:\n{}", body, make_indent(indent), statements_to_string(indent, orelse))
+    }
+}
+
+impl expr {
     fn to_string(&self) -> String {
         match self.clone() {
-            Expr::UnaryOp(op, operand) => format!("{}{}", op.to_string(), operand.to_string()),
-            Expr::BinOp(a, op, b) => format!("{} {} {}", a.to_string(), op.to_string(), b.to_string()),
-            Expr::Compare(left, ops, comparators) => format!("{} {}", left.to_string(), {
+            expr::BoolOp(op, values) => format!("{}({})", op.to_string(), args_to_string(values)),
+            expr::BinOp(a, op, b) => format!("{} {} {}", a.to_string(), op.to_string(), b.to_string()),
+            expr::UnaryOp(op, operand) => format!("{}{}", op.to_string(), operand.to_string()),
+
+            expr::Compare(left, ops, comparators) => format!("{} {}", left.to_string(), {
                 let mut arguments = vec!();
 
                 // XXX: wrong order!
@@ -52,142 +130,60 @@ impl Expr {
 
                 arguments.join(" ")
             }),
-            Expr::Call(func, args) => format!("{}({})", func.to_string(), {
-                let mut arguments = vec!();
-                for arg in args {
-                    arguments.push(arg.to_string());
-                }
-                arguments.join(", ")
-            }),
-            Expr::Alias(name, asname) => {
-                match asname {
-                    None => format!("{}", name),
-                    Some(asname) => format!("{} as {}", name, asname)
-                }
-            }
-            Expr::Attribute(lhs, rhs) => format!("{}.{}", lhs.to_string(), rhs),
-            Expr::Name(name) => format!("{}", name),
-            Expr::NameConstant(name) => format!("{}", name),
-            Expr::Str(s) => format!("\"{}\"", s),
-            Expr::Num(n) => format!("{}", n),
-            Expr::List(elts) => format!("[{}]", {
-                let mut elements = vec!();
-                for elt in elts {
-                    elements.push(elt.to_string());
-                }
-                elements.join(", ")
-            }),
-            Expr::Error => "Expr::Error".to_string()
+            expr::Call(func, args, keywords) => format!("{}({}{})", func.to_string(), args_to_string(args), kwargs_to_string(keywords)),
+            expr::Num(n) => format!("{}", n),
+            expr::Str(s) => format!("\"{}\"", s),
+            expr::NameConstant(name) => format!("{}", name),
+            expr::Attribute(value, attr, ctx) => format!("{}.{}", value.to_string(), attr),
+            expr::Name(name, ctx) => format!("{}", name),
+            expr::List(elts, ctx) => format!("[{}]", args_to_string(elts))
         }
     }
 }
 
+impl arguments {
+    fn to_string(&self) -> String {
+        let mut args = vec!();
+        for arg in self.args.clone() {
+            args.push(arg.arg);
+        }
+        args.join(", ")
+    }
+}
+
 fn make_indent(indent: usize) -> String {
-    let indent: String = iter::repeat("    ").take(indent).collect();
-    indent
+    iter::repeat("    ").take(indent).collect()
 }
 
-impl Statement {
+impl stmt {
     fn to_string(&self, indent: usize) -> String {
+        let current_indent = make_indent(indent);
         match self.clone() {
-            Statement::ClassDef(name, classes, body) => format!("{}def {}({}):\n{}", make_indent(indent), name, {
-                let mut bases = vec!();
-                for class in classes {
-                    bases.push(class.to_string());
-                }
-                bases.join(", ")
-            }, {
-                let mut statements = vec!();
-                for statement in body {
-                    statements.push(statement.to_string(indent + 1));
-                }
-                statements.join("\n")
-            }),
-            Statement::FunctionDef(name, arguments, body) => format!("{}def {}({}):\n{}", make_indent(indent), name, {
-                let mut args = vec!();
-                for arg in arguments {
-                    args.push(arg.to_string());
-                }
-                args.join(", ")
-            }, {
-                let mut statements = vec!();
-                for statement in body {
-                    statements.push(statement.to_string(indent + 1));
-                }
-                statements.join("\n")
+            stmt::ClassDef(name, bases, keywords, body, decorator_list) => format!("{}def {}({}):\n{}", current_indent, name, args_to_string(bases), statements_to_string(indent, body)),
+            stmt::FunctionDef(name, arguments, body, decorator_list, returns) => format!("{}def {}({}):\n{}", current_indent, name, arguments_to_string(arguments), statements_to_string(indent, body)),
+            stmt::Global(names) => format!("{}global {}", current_indent, names.join(", ")),
+            stmt::If(test, body, orelse) => format!("{}if {}:\n{}", current_indent, test.to_string(), if_else_statements_to_string(indent, body, orelse)),
+            stmt::While(test, body, orelse) => format!("{}while {}:\n{}", current_indent, test.to_string(), if_else_statements_to_string(indent, body, orelse)),
+            stmt::For(target, iter, body, orelse) => format!("{}for {} in {}:\n{}", current_indent, target.to_string(), iter.to_string(), if_else_statements_to_string(indent, body, orelse)),
+            stmt::Assign(targets, value) => format!("{}{} = {}", current_indent, args_to_string(targets), value.to_string()),
+            stmt::AugAssign(target, op, value) => format!("{}{} {}= {}", current_indent, target.to_string(), op.to_string(), value.to_string()),
+            stmt::Return(expr) => format!("{}return{}", current_indent, match expr {
+                Some(expr) => format!(" {}", expr.to_string()),
+                None => "".to_string()
             }),
-            Statement::Global(names) => format!("{}global {}", make_indent(indent), names.join(", ")),
-            Statement::If(test, body, orelse) => format!("{}if {}:\n{}", make_indent(indent), test.to_string(), {
-                let mut statements = vec!();
-                for arg in body {
-                    statements.push(arg.to_string(indent + 1));
-                }
-
-                let mut orelse_ = vec!();
-                for arg in orelse {
-                    orelse_.push(arg.to_string(indent + 1));
-                }
-
-                if orelse_.is_empty() {
-                    statements.join("\n")
-                } else {
-                    format!("{}\n{}else:\n{}", statements.join("\n"), make_indent(indent), orelse_.join("\n"))
-                }
-            }),
-            Statement::While(test, body, orelse) => format!("{}while {}:\n{}", make_indent(indent), test.to_string(), {
-                let mut statements = vec!();
-                for arg in body {
-                    statements.push(arg.to_string(indent + 1));
-                }
-
-                let mut orelse_ = vec!();
-                for arg in orelse {
-                    orelse_.push(arg.to_string(indent + 1));
-                }
-
-                if orelse_.is_empty() {
-                    statements.join("\n")
-                } else {
-                    format!("{}\n{}else:\n{}", statements.join("\n"), make_indent(indent), orelse_.join("\n"))
-                }
-            }),
-            Statement::For(target, iter, body, orelse) => format!("{}for {} in {}:\n{}", make_indent(indent), target.to_string(), iter.to_string(), {
-                let mut statements = vec!();
-                for arg in body {
-                    statements.push(arg.to_string(indent + 1));
-                }
-
-                let mut orelse_ = vec!();
-                for arg in orelse {
-                    orelse_.push(arg.to_string(indent + 1));
-                }
-
-                if orelse_.is_empty() {
-                    statements.join("\n")
-                } else {
-                    format!("{}\n{}else:\n{}", statements.join("\n"), make_indent(indent), orelse_.join("\n"))
-                }
-            }),
-            Statement::Assign(targets, value) => format!("{}{} = {}", make_indent(indent), {
-                let mut exprs = vec!();
-                for target in targets {
-                    exprs.push(target.to_string());
-                }
-                exprs.join(", ")
-            }, value.to_string()),
-            Statement::AugAssign(target, op, value) => format!("{}{} {}= {}", make_indent(indent), target.to_string(), op.to_string(), value.to_string()),
-            Statement::Return(expr) => format!("{}return {}", make_indent(indent), expr.to_string()),
-            Statement::ImportFrom(module, names) => format!("{}from {} import {}", make_indent(indent), module.to_string(), {
+            stmt::ImportFrom(module, names, level) => format!("{}from {} import {}", current_indent, module.to_string(), {
+                /*
                 let mut exprs = vec!();
                 for alias in names.iter() {
                     let alias = alias.to_string();
                     exprs.push(alias);
                 }
                 exprs.join(", ")
+                */
+                "".to_string()
             }),
-            Statement::Expr(expr) => format!("{}{}", make_indent(indent), expr.to_string()),
-            Statement::Break => format!("{}break", make_indent(indent)),
-            Statement::Error => format!("{}Statement::Error", make_indent(indent))
+            stmt::Expr(expr) => format!("{}{}", current_indent, expr.to_string()),
+            stmt::Break => format!("{}break", current_indent)
         }
     }
 }
--- a/src/ast_scope.rs
+++ b/src/ast_scope.rs
@@ -1,10 +1,10 @@
-use python_ast::{Module, Statement, Expr};
+use python_ast::{Module, stmt, expr, alias};
 
 use std::collections::HashMap;
 
 #[derive(Debug)]
 struct BlockStatement {
-    statement: Statement,
+    statement: stmt,
     block: Option<Block>
 }
 
@@ -19,43 +19,41 @@ pub struct Scoping {
     modules: HashMap<String, Block>
 }
 
-fn scope_expr(expr: Expr, block: &mut Block) {
+fn scope_expr(expr: expr, block: &mut Block) {
     println!("{:?}", expr);
 }
 
-fn scope_statement(statement: Statement, block: &mut Block) {
+fn scope_statement(statement: stmt, block: &mut Block) {
     let new_block = match statement.clone() {
-        Statement::Assign(targets, value) => {
+        stmt::Assign(targets, value) => {
             //scope_expr(value, &mut block);
             for target in targets {
                 match target {
-                    Expr::Name(name) => block.bindings.push(name),
+                    expr::Name(name, ctx) => block.bindings.push(name),
                     _ => ()  // No new binding.
                 }
             }
             None
         },
-        Statement::FunctionDef(name, args, statements) => {
+        stmt::FunctionDef(name, args, statements) => {
             block.bindings.push(name.clone());
-            let mut function_block = Block{statements: vec!(), bindings: args.clone()};
+            let mut function_block = Block{statements: vec!(), bindings: vec!()};
             for statement in statements {
                 scope_statement(statement, &mut function_block);
             }
             Some(function_block)
         },
-        Statement::ImportFrom(module, names) => {
-            for name in names {
-                let name = match name {
-                    Expr::Alias(name, asname) => (name, asname),
-                    _ => panic!()
-                };
-                let nameas = name.1;
-                let name = name.0;
-                match nameas {
+        stmt::ImportFrom(module, names, level) => {
+            /*
+            for alias in names {
+                let asname = alias.asname;
+                let name = alias.name;
+                match asname {
                     Some(name) => block.bindings.push(name),
                     None => block.bindings.push(name)
                 }
             }
+            */
             None
         },
         _ => None
@@ -75,3 +73,75 @@ pub fn scope_ast(modules: Vec<Module>) -
     }
     scoping
 }
+
+/*
+#[derive(Debug)]
+struct NewBlockStatement {
+    statement: stmt,
+    block: Option<NewBlock>
+}
+
+#[derive(Debug)]
+enum Binding {
+    Bound,
+    Free
+}
+
+#[derive(Debug)]
+struct NewBlock {
+    statements: Vec<NewBlockStatement>,
+    bindings: Vec<String>,
+    variables: HashMap<String, Binding>
+}
+
+#[derive(Debug)]
+pub struct NewScoping {
+    modules: HashMap<String, NewBlock>
+}
+
+fn check_expr(expr: expr, block: Block) {
+    println!("{:?}", expr);
+}
+
+fn check_block(block: Block) -> NewBlock {
+    let mut variables = HashMap::new();
+    let mut statements = vec!();
+    for blocked_statement in block.statements {
+        let statement = match blocked_statement.statement {
+            stmt::FunctionDef(name, args, body) => {
+                // No need to recurse here, this will be in a new block.
+                variables.insert(name.clone(), Binding::Bound);
+                stmt::FunctionDef(name, args, body)
+            },
+            stmt::Assign(targets, value) => {
+                for target in targets.clone() {
+                    match target {
+                        expr::Name(name) => {
+                            variables.insert(name, Binding::Bound);
+                        },
+                        _ => panic!()
+                    }
+                }
+                stmt::Assign(targets, value)
+            },
+            any => any
+        };
+        let new_block = match blocked_statement.block {
+            Some(block) => Some(check_block(block)),
+            None => None
+        };
+        statements.push(NewBlockStatement{statement: statement, block: new_block});
+    }
+    NewBlock{statements: statements, bindings: block.bindings, variables: variables}
+}
+
+#[allow(dead_code)]
+pub fn check_scoping(scoping: Scoping) -> NewScoping {
+    let mut modules = HashMap::new();
+    for (name, block) in scoping.modules {
+        modules.insert(name, check_block(block));
+    }
+    let mut new_scoping = NewScoping{modules: modules};
+    new_scoping
+}
+*/
--- a/src/main.rs
+++ b/src/main.rs
@@ -5,8 +5,8 @@ mod python_parse;
 mod python_dump;
 mod python_ast;
 mod ast_convert;
-mod ast_scope;
 mod ast_dump;
+//mod ast_scope;
 //mod ast_rewrite;
 //mod ast_type;
 
@@ -50,8 +50,10 @@ fn main() {
     //python_dump::dump_module(&module);
     let module_ast = ast_convert::convert_ast("__main__".to_string(), &module);
     ast_dump::dump_ast(&module_ast);
-    let scoped_ast = ast_scope::scope_ast(vec!(module_ast));
-    println!("{:#?}", scoped_ast);
+    //let scoped_ast = ast_scope::scope_ast(vec!(module_ast));
+    //println!("{:#?}", scoped_ast);
+    //let new_scoped_ast = ast_scope::check_scoping(scoped_ast);
+    //println!("{:#?}", new_scoped_ast);
     //ast_rewrite::rewrite_ast(module_ast);
     //ast_type::type_ast(scoped_ast);
 }
--- a/src/python_ast.rs
+++ b/src/python_ast.rs
@@ -1,57 +1,225 @@
 #[derive(Clone, Debug)]
 pub struct Module {
     pub name: String,
-    pub statements: Vec<Statement>,
+    pub statements: Vec<stmt>,
 }
 
-#[derive(Clone, Debug)]
-pub enum Statement {
-    ClassDef(String, Vec<Expr>, Vec<Statement>),
-    FunctionDef(String, Vec<String>, Vec<Statement>),
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum stmt {
+    // FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns)
+    FunctionDef(String, arguments, Vec<stmt>, Vec<expr>, Option<expr>),
+
+    // AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns)
+    //AsyncFunctionDef(String/*, arguments, Vec<expr>),
+
+    // ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list)
+    ClassDef(String, Vec<expr>, Vec<keyword>, Vec<stmt>, Vec<expr>),
+
+    // Return(expr? value)
+    Return(Option<expr>),
+
+    // Delete(expr* targets)
+    //Delete(Vec<expr>),
+
+    // Assign(expr* targets, expr value)
+    Assign(Vec<expr>, expr),
+
+    //AugAssign(expr target, operator op, expr value)
+    AugAssign(expr, operator, expr),
+
+    // For(expr target, expr iter, stmt* body, stmt* orelse)
+    For(expr, expr, Vec<stmt>, Vec<stmt>),
+
+    // AsyncFor(expr target, expr iter, stmt* body, stmt* orelse)
+    //AsyncFor(expr, expr, Vec<stmt>, Vec<stmt>),
+
+    // While(expr test, stmt* body, stmt* orelse)
+    While(expr, Vec<stmt>, Vec<stmt>),
+
+    // If(expr test, stmt* body, stmt* orelse)
+    If(expr, Vec<stmt>, Vec<stmt>),
+
+    // With(withitem* items, stmt* body)
+    //With(Vec<withitem>, Vec<stmt>)
+
+    // AsyncWith(withitem* items, stmt* body)
+    //AsyncWith(Vec<withitem>, Vec<stmt>)
+
+    // Raise(expr? exc, expr? cause)
+    //Raise(Option<expr>, Option<expr>)
+
+    // Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
+    //Try(Vec<stmt>, Vec<excepthandler>, Vec<stmt>, Vec<stmt>)
+
+    // Assert(expr test, expr? msg)
+    //Assert(expr, Option<expr>)
+
+    // Import(alias* names)
+    //Import(Vec<alias>)
+
+    // ImportFrom(identifier? module, alias* names, int? level)
+    ImportFrom(String, Vec<alias>, Option<i32>),  // TODO: check the size of level.
+
+    // Global(identifier* names)
     Global(Vec<String>),
-    If(Expr, Vec<Statement>, Vec<Statement>),
-    For(Expr, Expr, Vec<Statement>, Vec<Statement>),
-    While(Expr, Vec<Statement>, Vec<Statement>),
-    Assign(Vec<Expr>, Expr),
-    AugAssign(Expr, BinOp, Expr),
-    Return(Expr),
-    ImportFrom(String, Vec<Expr>),
-    Expr(Expr),
+
+    // Nonlocal(identifier* names)
+    //Nonlocal(Vec<String>),
+
+    // expr(expr value)
+    Expr(expr),
+
+    // Pass | Break | Continue
+    //Pass,
     Break,
-    Error
+    //Continue
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub enum Expr {
-    UnaryOp(UnaryOp, Box<Expr>),
-    BinOp(Box<Expr>, BinOp, Box<Expr>),
-    Compare(Box<Expr>, Vec<BinOp>, Vec<Expr>),
-    Call(Box<Expr>, Vec<Expr>),
-    Alias(String, Option<String>),
-    Attribute(Box<Expr>, String),
-    Name(String),
+pub enum expr {
+    BoolOp(boolop, Vec<expr>),
+    BinOp(Box<expr>, operator, Box<expr>),
+    UnaryOp(unaryop, Box<expr>),
+    //Lambda(arguments, Box<expr>)
+    //IfExp(Box<expr>, Box<expr>, Box<expr>)
+    //Dict(Vec<expr>, Vec<expr>)
+    //Set(Vec<expr>)
+    //ListComp(Box<expr>, Vec<comprehension>),
+    //SetComp(Box<expr>, Vec<comprehension>)
+    //DictComp(Box<expr>, Box<expr>, Vec<comprehension>)
+    //GeneratorExp(Box<expr>, Vec<comprehension>)
+
+    // the grammar constrains where yield expressions can occur
+    //Await(expr value)
+    //Yield(expr? value)
+    //YieldFrom(expr value)
+
+    // need sequences for compare to distinguish between
+    // x < 4 < 3 and (x < 4) < 3
+    Compare(Box<expr>, Vec<cmpop>, Vec<expr>),
+
+    Call(Box<expr>, Vec<expr>, Vec<keyword>),
+    Num(String),
+    Str(String),
+    //Bytes(String),
     NameConstant(String),
-    Str(String),
-    Num(String),
-    List(Vec<Expr>),
-    Error
+    //Ellipsis,
+
+    // the following expression can appear in assignment context
+    Attribute(Box<expr>, String, expr_context),
+    //Subscript(Box<expr>, slice, expr_context),
+    //Starred(Box<expr>, expr_context),
+    Name(String, expr_context),
+    List(Vec<expr>, expr_context),
+    //Tuple(Vec<expr>, expr_context),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum expr_context {
+    Load,
+    Store,
+    Del,
+    AugLoad,
+    AugStore,
+    Param
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum slice {
+    Slice(Option<expr>, Option<expr>, Option<expr>),
+    ExtSlice(Vec<slice>),
+    Index(expr)
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum boolop {
+    And,
+    Or
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub enum UnaryOp {
+pub enum operator {
+    Add,
+    Sub,
+    Mult,
+    MatMult,
+    Div,
+    Mod,
+    Pow,
+    LShift,
+    RShift,
+    BitOr,
+    BitXor,
+    BitAnd,
+    FloorDiv
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum unaryop {
+    Invert,
+    Not,
     UAdd,
-    USub,
-    Error
+    USub
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum cmpop {
+    Eq,
+    NotEq,
+    Lt,
+    LtE,
+    Gt,
+    GtE,
+    Is,
+    IsNot,
+    In,
+    NotIn
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub enum BinOp {
-    BinAdd,
-    BinMult,
-    BinEq,
-    BinLt,
-    BinGt,
-    Sub,
-    Div,
-    Error
+pub struct comprehension {
+    pub target: expr,
+    pub iter: expr,
+    pub ifs: Vec<expr>
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum excepthandler {
+    ExceptHandler(Option<expr>, Option<String>, Vec<stmt>)
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct arguments {
+    pub args: Vec<arg>,
+    pub vararg: Option<arg>,
+    pub kwonlyargs: Vec<arg>,
+    pub kw_defaults: Vec<expr>,
+    pub kwarg: Option<arg>,
+    pub defaults: Vec<expr>
 }
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct arg {
+    pub arg: String,
+    pub annotation: Option<expr>
+}
+
+// keyword arguments supplied to call (NULL identifier for **kwargs)
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct keyword {
+    pub arg: Option<String>,
+    pub value: expr
+}
+
+// import name with optional 'as' alias.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct alias {
+    pub name: String,
+    pub asname: Option<String>
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct withitem {
+    pub context_expr: expr,
+    pub optional_vars: Option<expr>
+}