Mercurial > python-compiler.rs
changeset 13:38b0d63697b1
Import the full AST grammar from CPython 3.5.
author | Emmanuel Gil Peyrot <linkmauve@linkmauve.fr> |
---|---|
date | Thu, 02 Jun 2016 05:33:23 +0100 |
parents | 0e96c5bc401d |
children | 719a27f1c1c7 |
files | src/ast_convert.rs src/ast_dump.rs src/ast_scope.rs src/main.rs src/python_ast.rs |
diffstat | 5 files changed, 612 insertions(+), 279 deletions(-) [+] |
line wrap: on
line diff
--- a/src/ast_convert.rs +++ b/src/ast_convert.rs @@ -1,4 +1,4 @@ -use python_ast::{Module, Statement, Expr, BinOp, UnaryOp}; +use python_ast::{Module, stmt, expr, expr_context, cmpop, operator, unaryop, arguments, arg, alias}; use cpython::{Python, PyObject}; use cpython::ObjectProtocol; //for call method @@ -30,7 +30,7 @@ fn parse_expr_vec(py: Python, ast: PyObj let arg = arg.unwrap(); let arg = parse_expr(py, arg); arguments.push(match arg { - Expr::Name(arg) => arg, + expr::Name(arg, expr_context::Load) => arg, _ => panic!() }); } @@ -40,7 +40,7 @@ fn parse_expr_vec(py: Python, ast: PyObj } } -fn parse_unaryop(py: Python, ast: PyObject) -> UnaryOp { +fn parse_unaryop(py: Python, ast: PyObject) -> unaryop { let builtins_module = py.import("builtins").unwrap(); let isinstance = builtins_module.get(py, "isinstance").unwrap(); @@ -50,22 +50,57 @@ fn parse_unaryop(py: Python, ast: PyObje let ast_module = py.import("ast").unwrap(); let ast_type = ast_module.get(py, "AST").unwrap(); + let invert_type = ast_module.get(py, "Invert").unwrap(); + let not_type = ast_module.get(py, "Not").unwrap(); let uadd_type = ast_module.get(py, "UAdd").unwrap(); let usub_type = ast_module.get(py, "USub").unwrap(); assert!(is_instance(&ast, &ast_type)); - if is_instance(&ast, &uadd_type) { - UnaryOp::UAdd + if is_instance(&ast, &invert_type) { + unaryop::Invert + } else if is_instance(&ast, ¬_type) { + unaryop::Not + } else if is_instance(&ast, &uadd_type) { + unaryop::UAdd } else if is_instance(&ast, &usub_type) { - UnaryOp::USub + unaryop::USub } else { - println!("UnaryOp {}", ast); - UnaryOp::Error + unreachable!() } } -fn parse_binop(py: Python, ast: PyObject) -> BinOp { +fn parse_cmpop(py: Python, ast: PyObject) -> cmpop { + let builtins_module = py.import("builtins").unwrap(); + let isinstance = builtins_module.get(py, "isinstance").unwrap(); + + let is_instance = |object: &PyObject, type_: &PyObject| { + return isinstance.call(py, (object, type_), None).unwrap().is_true(py).unwrap(); + }; + + let ast_module = py.import("ast").unwrap(); + let ast_type = ast_module.get(py, "AST").unwrap(); + let eq_type = ast_module.get(py, "Eq").unwrap(); + let noteq_type = ast_module.get(py, "NotEq").unwrap(); + let lt_type = ast_module.get(py, "Lt").unwrap(); + let gt_type = ast_module.get(py, "Gt").unwrap(); + + assert!(is_instance(&ast, &ast_type)); + + if is_instance(&ast, &eq_type) { + cmpop::Eq + } else if is_instance(&ast, ¬eq_type) { + cmpop::NotEq + } else if is_instance(&ast, <_type) { + cmpop::Lt + } else if is_instance(&ast, >_type) { + cmpop::Gt + } else { + unreachable!() + } +} + +fn parse_operator(py: Python, ast: PyObject) -> operator { let builtins_module = py.import("builtins").unwrap(); let isinstance = builtins_module.get(py, "isinstance").unwrap(); @@ -76,36 +111,54 @@ fn parse_binop(py: Python, ast: PyObject let ast_module = py.import("ast").unwrap(); let ast_type = ast_module.get(py, "AST").unwrap(); let add_type = ast_module.get(py, "Add").unwrap(); + let sub_type = ast_module.get(py, "Sub").unwrap(); let mult_type = ast_module.get(py, "Mult").unwrap(); - let eq_type = ast_module.get(py, "Eq").unwrap(); - let lt_type = ast_module.get(py, "Lt").unwrap(); - let gt_type = ast_module.get(py, "Gt").unwrap(); - let sub_type = ast_module.get(py, "Sub").unwrap(); + let matmult_type = ast_module.get(py, "MatMult").unwrap(); let div_type = ast_module.get(py, "Div").unwrap(); + let mod_type = ast_module.get(py, "Mod").unwrap(); + let pow_type = ast_module.get(py, "Pow").unwrap(); + let lshift_type = ast_module.get(py, "LShift").unwrap(); + let rshift_type = ast_module.get(py, "RShift").unwrap(); + let bitor_type = ast_module.get(py, "BitOr").unwrap(); + let bitxor_type = ast_module.get(py, "BitXor").unwrap(); + let bitand_type = ast_module.get(py, "BitAnd").unwrap(); + let floordiv_type = ast_module.get(py, "FloorDiv").unwrap(); assert!(is_instance(&ast, &ast_type)); if is_instance(&ast, &add_type) { - BinOp::BinAdd + operator::Add + } else if is_instance(&ast, &sub_type) { + operator::Sub } else if is_instance(&ast, &mult_type) { - BinOp::BinMult - } else if is_instance(&ast, &eq_type) { - BinOp::BinEq - } else if is_instance(&ast, <_type) { - BinOp::BinLt - } else if is_instance(&ast, >_type) { - BinOp::BinGt - } else if is_instance(&ast, &sub_type) { - BinOp::Sub + operator::Mult + } else if is_instance(&ast, &matmult_type) { + operator::MatMult } else if is_instance(&ast, &div_type) { - BinOp::Div + operator::Div + } else if is_instance(&ast, &mod_type) { + operator::Mod + } else if is_instance(&ast, &pow_type) { + operator::Pow + } else if is_instance(&ast, &lshift_type) { + operator::LShift + } else if is_instance(&ast, &rshift_type) { + operator::RShift + } else if is_instance(&ast, &bitor_type) { + operator::BitOr + } else if is_instance(&ast, &bitxor_type) { + operator::BitXor + } else if is_instance(&ast, &bitand_type) { + operator::BitAnd + } else if is_instance(&ast, &floordiv_type) { + operator::FloorDiv } else { - println!("BinOp {}", ast); - BinOp::Error + println!("operator {}", ast); + panic!() } } -fn parse_expr(py: Python, ast: PyObject) -> Expr { +fn parse_expr(py: Python, ast: PyObject) -> expr { let builtins_module = py.import("builtins").unwrap(); let isinstance = builtins_module.get(py, "isinstance").unwrap(); @@ -126,14 +179,13 @@ fn parse_expr(py: Python, ast: PyObject) let list_type = ast_module.get(py, "List").unwrap(); let compare_type = ast_module.get(py, "Compare").unwrap(); let call_type = ast_module.get(py, "Call").unwrap(); - let alias_type = ast_module.get(py, "alias").unwrap(); assert!(is_instance(&ast, &ast_type)); if is_instance(&ast, &arg_type) { let arg = ast.getattr(py, "arg").unwrap(); let arg = get_str(py, arg); - Expr::Name(arg) + expr::Name(arg, expr_context::Load) } else if is_instance(&ast, &attribute_type) { let value = ast.getattr(py, "value").unwrap(); let attr = ast.getattr(py, "attr").unwrap(); @@ -141,23 +193,23 @@ fn parse_expr(py: Python, ast: PyObject) let value = parse_expr(py, value); let attr = get_str(py, attr); - Expr::Attribute(Box::new(value), attr) + expr::Attribute(Box::new(value), attr, expr_context::Load) } else if is_instance(&ast, &name_type) { let id = ast.getattr(py, "id").unwrap(); let id = get_str(py, id); - Expr::Name(id) + expr::Name(id, expr_context::Load) } else if is_instance(&ast, &name_constant_type) { let value = ast.getattr(py, "value").unwrap(); let value = get_str(py, value); - Expr::NameConstant(value) + expr::NameConstant(value) } else if is_instance(&ast, &num_type) { let n = ast.getattr(py, "n").unwrap(); let n = get_str(py, n); - Expr::Num(n) + expr::Num(n) } else if is_instance(&ast, &str_type) { let s = ast.getattr(py, "s").unwrap(); let s = get_str(py, s); - Expr::Str(s) + expr::Str(s) } else if is_instance(&ast, &list_type) { let elts = ast.getattr(py, "elts").unwrap(); @@ -167,7 +219,7 @@ fn parse_expr(py: Python, ast: PyObject) elements.push(parse_expr(py, elt)); } - Expr::List(elements) + expr::List(elements, expr_context::Load) } else if is_instance(&ast, &unary_op_type) { let op = ast.getattr(py, "op").unwrap(); let operand = ast.getattr(py, "operand").unwrap(); @@ -175,21 +227,21 @@ fn parse_expr(py: Python, ast: PyObject) let op = parse_unaryop(py, op); let operand = parse_expr(py, operand); - Expr::UnaryOp(op, Box::new(operand)) + expr::UnaryOp(op, Box::new(operand)) } else if is_instance(&ast, &bin_op_type) { let left = ast.getattr(py, "left").unwrap(); let op = ast.getattr(py, "op").unwrap(); let right = ast.getattr(py, "right").unwrap(); let left = parse_expr(py, left); - let op = parse_binop(py, op); + let op = parse_operator(py, op); let right = parse_expr(py, right); - Expr::BinOp(Box::new(left), op, Box::new(right)) + expr::BinOp(Box::new(left), op, Box::new(right)) } else if is_instance(&ast, &call_type) { let func = ast.getattr(py, "func").unwrap(); let args = ast.getattr(py, "args").unwrap(); - //let keywords = ast.getattr(py, "keywords").unwrap(); + let keywords = ast.getattr(py, "keywords").unwrap(); let func = parse_expr(py, func); @@ -199,19 +251,15 @@ fn parse_expr(py: Python, ast: PyObject) arguments.push(parse_expr(py, arg)); } - Expr::Call(Box::new(func), arguments) - } else if is_instance(&ast, &alias_type) { - let name = ast.getattr(py, "name").unwrap(); - let asname = ast.getattr(py, "asname").unwrap(); + /* + let mut kwargs = vec!(); + for arg in keywords.iter(py).unwrap() { + let arg = arg.unwrap(); + kwargs.push(parse_expr(py, arg)); + } + */ - let name = get_str(py, name); - let asname = if asname == py.None() { - None - } else { - Some(get_str(py, asname)) - }; - - Expr::Alias(name, asname) + expr::Call(Box::new(func), arguments, vec!()) } else if is_instance(&ast, &compare_type) { let left = ast.getattr(py, "left").unwrap(); let ops = ast.getattr(py, "ops").unwrap(); @@ -224,7 +272,7 @@ fn parse_expr(py: Python, ast: PyObject) let mut new_ops = vec!(); for op in ops { let op = op.unwrap(); - let op = parse_binop(py, op); + let op = parse_cmpop(py, op); new_ops.push(op); } @@ -235,16 +283,63 @@ fn parse_expr(py: Python, ast: PyObject) new_comparators.push(comparator); } - Expr::Compare(Box::new(left), new_ops, new_comparators) + expr::Compare(Box::new(left), new_ops, new_comparators) } else { - println!("Expr {}", ast); - Expr::Error + println!("expr {}", ast); + unreachable!() } } -fn parse_statement(py: Python, ast: PyObject) -> Statement { - //Statement::FunctionDef(Expr::Name("function".to_string()), vec!(Expr::Name("a".to_string()), Expr::Name("b".to_string())), vec!()) - //Statement::If(Expr::BinOp(BinOp::BinEq, Box::new(Expr::Name("__name__".to_string())), Box::new(Expr::Str("__main__".to_string()))), vec!(Statement::Expr(Expr::Call(Box::new(Expr::Name("function".to_string())), vec!(Expr::Num(1), Expr::Num(2)))))) +fn parse_arguments(py: Python, ast: PyObject) -> arguments { + let builtins_module = py.import("builtins").unwrap(); + let isinstance = builtins_module.get(py, "isinstance").unwrap(); + + let is_instance = |object: &PyObject, type_: &PyObject| { + return isinstance.call(py, (object, type_), None).unwrap().is_true(py).unwrap(); + }; + + let ast_module = py.import("ast").unwrap(); + let ast_type = ast_module.get(py, "AST").unwrap(); + let arguments_type = ast_module.get(py, "arguments").unwrap(); + let arg_type = ast_module.get(py, "arg").unwrap(); + + assert!(is_instance(&ast, &ast_type)); + + if is_instance(&ast, &arguments_type) { + let args = arguments{ + //args: Vec<arg>, + args: { + let args = ast.getattr(py, "args").unwrap(); + let mut arguments = vec!(); + for arg in args.iter(py).unwrap() { + let arg = arg.unwrap(); + assert!(is_instance(&arg, &arg_type)); + let arg = get_str(py, arg); + arguments.push(arg{arg: arg, annotation: None}); + } + arguments + }, + //vararg: Option<arg>, + vararg: None, + //kwonlyargs: Vec<arg>, + kwonlyargs: vec!(), + //kw_defaults: Vec<expr>, + kw_defaults: vec!(), + //kwarg: Option<arg>, + kwarg: None, + //defaults: Vec<expr> + defaults: vec!() + }; + args + } else { + println!("arguments {}", ast); + panic!() + } +} + +fn parse_statement(py: Python, ast: PyObject) -> stmt { + //stmt::FunctionDef(expr::Name("function".to_string()), vec!(expr::Name("a".to_string()), expr::Name("b".to_string())), vec!()) + //stmt::If(expr::BinOp(BinOp::BinEq, Box::new(expr::Name("__name__".to_string())), Box::new(expr::Str("__main__".to_string()))), vec!(stmt::Expr(expr::Call(Box::new(expr::Name("function".to_string())), vec!(expr::Num(1), expr::Num(2)))))) let builtins_module = py.import("builtins").unwrap(); let isinstance = builtins_module.get(py, "isinstance").unwrap(); @@ -307,21 +402,14 @@ fn parse_statement(py: Python, ast: PyOb statements.push(statement); } - Statement::ClassDef(name, nodes, statements) + stmt::ClassDef(name, nodes, vec!(), statements, vec!()) } else if is_instance(&ast, &function_def_type) { let name = ast.getattr(py, "name").unwrap(); let args = ast.getattr(py, "args").unwrap(); let body = ast.getattr(py, "body").unwrap(); let name = get_str(py, name); - let args = parse_expr_vec(py, args); - /* - let mut arguments = vec!(); - for arg in args.iter(py).unwrap() { - let arg = parse_expr(py, arg.unwrap()); - arguments.push(arg); - } - */ + let args = parse_arguments(py, args); let mut statements = vec!(); for statement in body.iter(py).unwrap() { @@ -329,7 +417,10 @@ fn parse_statement(py: Python, ast: PyOb statements.push(statement); } - Statement::FunctionDef(name, args, statements) + let decorators = vec!(); + let returns = None; + + stmt::FunctionDef(name, args, statements, decorators, returns) } else if is_instance(&ast, &global_type) { let names = ast.getattr(py, "names").unwrap(); @@ -340,7 +431,7 @@ fn parse_statement(py: Python, ast: PyOb globals.push(name); } - Statement::Global(globals) + stmt::Global(globals) } else if is_instance(&ast, &if_type) { let test = ast.getattr(py, "test").unwrap(); let body = ast.getattr(py, "body").unwrap(); @@ -362,7 +453,7 @@ fn parse_statement(py: Python, ast: PyOb orelse_.push(statement); } - Statement::If(test, statements, orelse_) + stmt::If(test, statements, orelse_) } else if is_instance(&ast, &while_type) { let test = ast.getattr(py, "test").unwrap(); let body = ast.getattr(py, "body").unwrap(); @@ -384,7 +475,7 @@ fn parse_statement(py: Python, ast: PyOb orelse_.push(statement); } - Statement::While(test, statements, orelse_) + stmt::While(test, statements, orelse_) } else if is_instance(&ast, &for_type) { let target = ast.getattr(py, "target").unwrap(); let iter = ast.getattr(py, "iter").unwrap(); @@ -408,7 +499,7 @@ fn parse_statement(py: Python, ast: PyOb orelse_.push(statement); } - Statement::For(target, iter, statements, orelse_) + stmt::For(target, iter, statements, orelse_) } else if is_instance(&ast, &assign_type) { let targets = ast.getattr(py, "targets").unwrap(); let value = ast.getattr(py, "value").unwrap(); @@ -422,21 +513,21 @@ fn parse_statement(py: Python, ast: PyOb let value = parse_expr(py, value); - Statement::Assign(arguments, value) + stmt::Assign(arguments, value) } else if is_instance(&ast, &aug_assign_type) { let target = ast.getattr(py, "target").unwrap(); let op = ast.getattr(py, "op").unwrap(); let value = ast.getattr(py, "value").unwrap(); let target = parse_expr(py, target); - let op = parse_binop(py, op); + let op = parse_operator(py, op); let value = parse_expr(py, value); - Statement::AugAssign(target, op, value) + stmt::AugAssign(target, op, value) } else if is_instance(&ast, &import_from_type) { let module = ast.getattr(py, "module").unwrap(); let names = ast.getattr(py, "names").unwrap(); - //let level = ast.getattr(py, "level").unwrap(); + let level = ast.getattr(py, "level").unwrap(); let module = get_str(py, module); @@ -444,23 +535,29 @@ fn parse_statement(py: Python, ast: PyOb for alias in names.iter(py).unwrap() { let alias = alias.unwrap(); let alias = parse_expr(py, alias); - names_.push(alias); + println!("{:?}", alias); + // XXX + //names_.push(alias{name: alias, asname: alias}); } - Statement::ImportFrom(module, names_) + stmt::ImportFrom(module, names_, None) } else if is_instance(&ast, &return_type) { let value = ast.getattr(py, "value").unwrap(); - let value = parse_expr(py, value); - Statement::Return(value) + if value == py.None() { + stmt::Return(None) + } else { + let value = parse_expr(py, value); + stmt::Return(Some(value)) + } } else if is_instance(&ast, &expr_type) { let value = ast.getattr(py, "value").unwrap(); let value = parse_expr(py, value); - Statement::Expr(value) + stmt::Expr(value) } else if is_instance(&ast, &break_type) { - Statement::Break + stmt::Break } else { - println!("Statement {}", ast); - Statement::Error + println!("stmt {}", ast); + panic!() } }
--- a/src/ast_dump.rs +++ b/src/ast_dump.rs @@ -1,38 +1,116 @@ -use python_ast::{Module, Statement, Expr, BinOp, UnaryOp}; +use python_ast::{Module, stmt, expr, boolop, operator, unaryop, cmpop, arguments, arg, keyword}; use std::iter; -impl UnaryOp { +impl boolop { + fn to_string(&self) -> &'static str { + match *self { + boolop::And => "and", + boolop::Or => "or" + } + } +} + +impl operator { fn to_string(&self) -> &'static str { match *self { - UnaryOp::UAdd => "+", - UnaryOp::USub => "-", - UnaryOp::Error => "BinOp::Error" + operator::Add => "+", + operator::Sub => "-", + operator::Mult => "*", + operator::MatMult => "@", + operator::Div => "/", + operator::Mod => "%", + operator::Pow => "**", + operator::LShift => "<<", + operator::RShift => ">>", + operator::BitOr => "|", + operator::BitXor => "^", + operator::BitAnd => "&", + operator::FloorDiv => "//", + } + } +} + +impl unaryop { + fn to_string(&self) -> &'static str { + match *self { + unaryop::Invert => "~", + unaryop::Not => "!", + unaryop::UAdd => "+", + unaryop::USub => "-" } } } -impl BinOp { +impl cmpop { fn to_string(&self) -> &'static str { match *self { - BinOp::BinAdd => "+", - BinOp::BinMult => "*", - BinOp::BinEq => "==", - BinOp::BinLt => "<", - BinOp::BinGt => ">", - BinOp::Sub => "-", - BinOp::Div => "/", - BinOp::Error => "BinOp::Error" + cmpop::Eq => "==", + cmpop::NotEq => "!=", + cmpop::Lt => "<", + cmpop::LtE => "<=", + cmpop::Gt => ">", + cmpop::GtE => ">=", + cmpop::Is => "is", + cmpop::IsNot => "is not", + cmpop::In => "in", + cmpop::NotIn => "not in", } } } -impl Expr { +fn args_to_string(args: Vec<expr>) -> String { + let mut arguments = vec!(); + for arg in args { + arguments.push(arg.to_string()); + } + arguments.join(", ") +} + +fn kwargs_to_string(kwargs: Vec<keyword>) -> String { + let mut arguments = vec!(); + for arg in kwargs { + match arg.arg { + Some(arg) => arguments.push(arg), + None => arguments.push("**kwarg".to_string()) + } + } + arguments.join(", ") +} + +fn arguments_to_string(args: arguments) -> String { + let mut arguments = vec!(); + for arg in args.args { + arguments.push(arg.arg); + } + arguments.join(", ") +} + +fn statements_to_string(indent: usize, body: Vec<stmt>) -> String { + let mut statements = vec!(); + for statement in body { + statements.push(statement.to_string(indent + 1)); + } + statements.join("\n") +} + +fn if_else_statements_to_string(indent: usize, body: Vec<stmt>, orelse: Vec<stmt>) -> String { + let body = statements_to_string(indent, body); + if orelse.is_empty() { + body + } else { + format!("{}\n{}else:\n{}", body, make_indent(indent), statements_to_string(indent, orelse)) + } +} + +impl expr { fn to_string(&self) -> String { match self.clone() { - Expr::UnaryOp(op, operand) => format!("{}{}", op.to_string(), operand.to_string()), - Expr::BinOp(a, op, b) => format!("{} {} {}", a.to_string(), op.to_string(), b.to_string()), - Expr::Compare(left, ops, comparators) => format!("{} {}", left.to_string(), { + expr::BoolOp(op, values) => format!("{}({})", op.to_string(), args_to_string(values)), + expr::BinOp(a, op, b) => format!("{} {} {}", a.to_string(), op.to_string(), b.to_string()), + expr::UnaryOp(op, operand) => format!("{}{}", op.to_string(), operand.to_string()), + + expr::Compare(left, ops, comparators) => format!("{} {}", left.to_string(), { let mut arguments = vec!(); // XXX: wrong order! @@ -52,142 +130,60 @@ impl Expr { arguments.join(" ") }), - Expr::Call(func, args) => format!("{}({})", func.to_string(), { - let mut arguments = vec!(); - for arg in args { - arguments.push(arg.to_string()); - } - arguments.join(", ") - }), - Expr::Alias(name, asname) => { - match asname { - None => format!("{}", name), - Some(asname) => format!("{} as {}", name, asname) - } - } - Expr::Attribute(lhs, rhs) => format!("{}.{}", lhs.to_string(), rhs), - Expr::Name(name) => format!("{}", name), - Expr::NameConstant(name) => format!("{}", name), - Expr::Str(s) => format!("\"{}\"", s), - Expr::Num(n) => format!("{}", n), - Expr::List(elts) => format!("[{}]", { - let mut elements = vec!(); - for elt in elts { - elements.push(elt.to_string()); - } - elements.join(", ") - }), - Expr::Error => "Expr::Error".to_string() + expr::Call(func, args, keywords) => format!("{}({}{})", func.to_string(), args_to_string(args), kwargs_to_string(keywords)), + expr::Num(n) => format!("{}", n), + expr::Str(s) => format!("\"{}\"", s), + expr::NameConstant(name) => format!("{}", name), + expr::Attribute(value, attr, ctx) => format!("{}.{}", value.to_string(), attr), + expr::Name(name, ctx) => format!("{}", name), + expr::List(elts, ctx) => format!("[{}]", args_to_string(elts)) } } } +impl arguments { + fn to_string(&self) -> String { + let mut args = vec!(); + for arg in self.args.clone() { + args.push(arg.arg); + } + args.join(", ") + } +} + fn make_indent(indent: usize) -> String { - let indent: String = iter::repeat(" ").take(indent).collect(); - indent + iter::repeat(" ").take(indent).collect() } -impl Statement { +impl stmt { fn to_string(&self, indent: usize) -> String { + let current_indent = make_indent(indent); match self.clone() { - Statement::ClassDef(name, classes, body) => format!("{}def {}({}):\n{}", make_indent(indent), name, { - let mut bases = vec!(); - for class in classes { - bases.push(class.to_string()); - } - bases.join(", ") - }, { - let mut statements = vec!(); - for statement in body { - statements.push(statement.to_string(indent + 1)); - } - statements.join("\n") - }), - Statement::FunctionDef(name, arguments, body) => format!("{}def {}({}):\n{}", make_indent(indent), name, { - let mut args = vec!(); - for arg in arguments { - args.push(arg.to_string()); - } - args.join(", ") - }, { - let mut statements = vec!(); - for statement in body { - statements.push(statement.to_string(indent + 1)); - } - statements.join("\n") + stmt::ClassDef(name, bases, keywords, body, decorator_list) => format!("{}def {}({}):\n{}", current_indent, name, args_to_string(bases), statements_to_string(indent, body)), + stmt::FunctionDef(name, arguments, body, decorator_list, returns) => format!("{}def {}({}):\n{}", current_indent, name, arguments_to_string(arguments), statements_to_string(indent, body)), + stmt::Global(names) => format!("{}global {}", current_indent, names.join(", ")), + stmt::If(test, body, orelse) => format!("{}if {}:\n{}", current_indent, test.to_string(), if_else_statements_to_string(indent, body, orelse)), + stmt::While(test, body, orelse) => format!("{}while {}:\n{}", current_indent, test.to_string(), if_else_statements_to_string(indent, body, orelse)), + stmt::For(target, iter, body, orelse) => format!("{}for {} in {}:\n{}", current_indent, target.to_string(), iter.to_string(), if_else_statements_to_string(indent, body, orelse)), + stmt::Assign(targets, value) => format!("{}{} = {}", current_indent, args_to_string(targets), value.to_string()), + stmt::AugAssign(target, op, value) => format!("{}{} {}= {}", current_indent, target.to_string(), op.to_string(), value.to_string()), + stmt::Return(expr) => format!("{}return{}", current_indent, match expr { + Some(expr) => format!(" {}", expr.to_string()), + None => "".to_string() }), - Statement::Global(names) => format!("{}global {}", make_indent(indent), names.join(", ")), - Statement::If(test, body, orelse) => format!("{}if {}:\n{}", make_indent(indent), test.to_string(), { - let mut statements = vec!(); - for arg in body { - statements.push(arg.to_string(indent + 1)); - } - - let mut orelse_ = vec!(); - for arg in orelse { - orelse_.push(arg.to_string(indent + 1)); - } - - if orelse_.is_empty() { - statements.join("\n") - } else { - format!("{}\n{}else:\n{}", statements.join("\n"), make_indent(indent), orelse_.join("\n")) - } - }), - Statement::While(test, body, orelse) => format!("{}while {}:\n{}", make_indent(indent), test.to_string(), { - let mut statements = vec!(); - for arg in body { - statements.push(arg.to_string(indent + 1)); - } - - let mut orelse_ = vec!(); - for arg in orelse { - orelse_.push(arg.to_string(indent + 1)); - } - - if orelse_.is_empty() { - statements.join("\n") - } else { - format!("{}\n{}else:\n{}", statements.join("\n"), make_indent(indent), orelse_.join("\n")) - } - }), - Statement::For(target, iter, body, orelse) => format!("{}for {} in {}:\n{}", make_indent(indent), target.to_string(), iter.to_string(), { - let mut statements = vec!(); - for arg in body { - statements.push(arg.to_string(indent + 1)); - } - - let mut orelse_ = vec!(); - for arg in orelse { - orelse_.push(arg.to_string(indent + 1)); - } - - if orelse_.is_empty() { - statements.join("\n") - } else { - format!("{}\n{}else:\n{}", statements.join("\n"), make_indent(indent), orelse_.join("\n")) - } - }), - Statement::Assign(targets, value) => format!("{}{} = {}", make_indent(indent), { - let mut exprs = vec!(); - for target in targets { - exprs.push(target.to_string()); - } - exprs.join(", ") - }, value.to_string()), - Statement::AugAssign(target, op, value) => format!("{}{} {}= {}", make_indent(indent), target.to_string(), op.to_string(), value.to_string()), - Statement::Return(expr) => format!("{}return {}", make_indent(indent), expr.to_string()), - Statement::ImportFrom(module, names) => format!("{}from {} import {}", make_indent(indent), module.to_string(), { + stmt::ImportFrom(module, names, level) => format!("{}from {} import {}", current_indent, module.to_string(), { + /* let mut exprs = vec!(); for alias in names.iter() { let alias = alias.to_string(); exprs.push(alias); } exprs.join(", ") + */ + "".to_string() }), - Statement::Expr(expr) => format!("{}{}", make_indent(indent), expr.to_string()), - Statement::Break => format!("{}break", make_indent(indent)), - Statement::Error => format!("{}Statement::Error", make_indent(indent)) + stmt::Expr(expr) => format!("{}{}", current_indent, expr.to_string()), + stmt::Break => format!("{}break", current_indent) } } }
--- a/src/ast_scope.rs +++ b/src/ast_scope.rs @@ -1,10 +1,10 @@ -use python_ast::{Module, Statement, Expr}; +use python_ast::{Module, stmt, expr, alias}; use std::collections::HashMap; #[derive(Debug)] struct BlockStatement { - statement: Statement, + statement: stmt, block: Option<Block> } @@ -19,43 +19,41 @@ pub struct Scoping { modules: HashMap<String, Block> } -fn scope_expr(expr: Expr, block: &mut Block) { +fn scope_expr(expr: expr, block: &mut Block) { println!("{:?}", expr); } -fn scope_statement(statement: Statement, block: &mut Block) { +fn scope_statement(statement: stmt, block: &mut Block) { let new_block = match statement.clone() { - Statement::Assign(targets, value) => { + stmt::Assign(targets, value) => { //scope_expr(value, &mut block); for target in targets { match target { - Expr::Name(name) => block.bindings.push(name), + expr::Name(name, ctx) => block.bindings.push(name), _ => () // No new binding. } } None }, - Statement::FunctionDef(name, args, statements) => { + stmt::FunctionDef(name, args, statements) => { block.bindings.push(name.clone()); - let mut function_block = Block{statements: vec!(), bindings: args.clone()}; + let mut function_block = Block{statements: vec!(), bindings: vec!()}; for statement in statements { scope_statement(statement, &mut function_block); } Some(function_block) }, - Statement::ImportFrom(module, names) => { - for name in names { - let name = match name { - Expr::Alias(name, asname) => (name, asname), - _ => panic!() - }; - let nameas = name.1; - let name = name.0; - match nameas { + stmt::ImportFrom(module, names, level) => { + /* + for alias in names { + let asname = alias.asname; + let name = alias.name; + match asname { Some(name) => block.bindings.push(name), None => block.bindings.push(name) } } + */ None }, _ => None @@ -75,3 +73,75 @@ pub fn scope_ast(modules: Vec<Module>) - } scoping } + +/* +#[derive(Debug)] +struct NewBlockStatement { + statement: stmt, + block: Option<NewBlock> +} + +#[derive(Debug)] +enum Binding { + Bound, + Free +} + +#[derive(Debug)] +struct NewBlock { + statements: Vec<NewBlockStatement>, + bindings: Vec<String>, + variables: HashMap<String, Binding> +} + +#[derive(Debug)] +pub struct NewScoping { + modules: HashMap<String, NewBlock> +} + +fn check_expr(expr: expr, block: Block) { + println!("{:?}", expr); +} + +fn check_block(block: Block) -> NewBlock { + let mut variables = HashMap::new(); + let mut statements = vec!(); + for blocked_statement in block.statements { + let statement = match blocked_statement.statement { + stmt::FunctionDef(name, args, body) => { + // No need to recurse here, this will be in a new block. + variables.insert(name.clone(), Binding::Bound); + stmt::FunctionDef(name, args, body) + }, + stmt::Assign(targets, value) => { + for target in targets.clone() { + match target { + expr::Name(name) => { + variables.insert(name, Binding::Bound); + }, + _ => panic!() + } + } + stmt::Assign(targets, value) + }, + any => any + }; + let new_block = match blocked_statement.block { + Some(block) => Some(check_block(block)), + None => None + }; + statements.push(NewBlockStatement{statement: statement, block: new_block}); + } + NewBlock{statements: statements, bindings: block.bindings, variables: variables} +} + +#[allow(dead_code)] +pub fn check_scoping(scoping: Scoping) -> NewScoping { + let mut modules = HashMap::new(); + for (name, block) in scoping.modules { + modules.insert(name, check_block(block)); + } + let mut new_scoping = NewScoping{modules: modules}; + new_scoping +} +*/
--- a/src/main.rs +++ b/src/main.rs @@ -5,8 +5,8 @@ mod python_parse; mod python_dump; mod python_ast; mod ast_convert; -mod ast_scope; mod ast_dump; +//mod ast_scope; //mod ast_rewrite; //mod ast_type; @@ -50,8 +50,10 @@ fn main() { //python_dump::dump_module(&module); let module_ast = ast_convert::convert_ast("__main__".to_string(), &module); ast_dump::dump_ast(&module_ast); - let scoped_ast = ast_scope::scope_ast(vec!(module_ast)); - println!("{:#?}", scoped_ast); + //let scoped_ast = ast_scope::scope_ast(vec!(module_ast)); + //println!("{:#?}", scoped_ast); + //let new_scoped_ast = ast_scope::check_scoping(scoped_ast); + //println!("{:#?}", new_scoped_ast); //ast_rewrite::rewrite_ast(module_ast); //ast_type::type_ast(scoped_ast); }
--- a/src/python_ast.rs +++ b/src/python_ast.rs @@ -1,57 +1,225 @@ #[derive(Clone, Debug)] pub struct Module { pub name: String, - pub statements: Vec<Statement>, + pub statements: Vec<stmt>, } -#[derive(Clone, Debug)] -pub enum Statement { - ClassDef(String, Vec<Expr>, Vec<Statement>), - FunctionDef(String, Vec<String>, Vec<Statement>), +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum stmt { + // FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns) + FunctionDef(String, arguments, Vec<stmt>, Vec<expr>, Option<expr>), + + // AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns) + //AsyncFunctionDef(String/*, arguments, Vec<expr>), + + // ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list) + ClassDef(String, Vec<expr>, Vec<keyword>, Vec<stmt>, Vec<expr>), + + // Return(expr? value) + Return(Option<expr>), + + // Delete(expr* targets) + //Delete(Vec<expr>), + + // Assign(expr* targets, expr value) + Assign(Vec<expr>, expr), + + //AugAssign(expr target, operator op, expr value) + AugAssign(expr, operator, expr), + + // For(expr target, expr iter, stmt* body, stmt* orelse) + For(expr, expr, Vec<stmt>, Vec<stmt>), + + // AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) + //AsyncFor(expr, expr, Vec<stmt>, Vec<stmt>), + + // While(expr test, stmt* body, stmt* orelse) + While(expr, Vec<stmt>, Vec<stmt>), + + // If(expr test, stmt* body, stmt* orelse) + If(expr, Vec<stmt>, Vec<stmt>), + + // With(withitem* items, stmt* body) + //With(Vec<withitem>, Vec<stmt>) + + // AsyncWith(withitem* items, stmt* body) + //AsyncWith(Vec<withitem>, Vec<stmt>) + + // Raise(expr? exc, expr? cause) + //Raise(Option<expr>, Option<expr>) + + // Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + //Try(Vec<stmt>, Vec<excepthandler>, Vec<stmt>, Vec<stmt>) + + // Assert(expr test, expr? msg) + //Assert(expr, Option<expr>) + + // Import(alias* names) + //Import(Vec<alias>) + + // ImportFrom(identifier? module, alias* names, int? level) + ImportFrom(String, Vec<alias>, Option<i32>), // TODO: check the size of level. + + // Global(identifier* names) Global(Vec<String>), - If(Expr, Vec<Statement>, Vec<Statement>), - For(Expr, Expr, Vec<Statement>, Vec<Statement>), - While(Expr, Vec<Statement>, Vec<Statement>), - Assign(Vec<Expr>, Expr), - AugAssign(Expr, BinOp, Expr), - Return(Expr), - ImportFrom(String, Vec<Expr>), - Expr(Expr), + + // Nonlocal(identifier* names) + //Nonlocal(Vec<String>), + + // expr(expr value) + Expr(expr), + + // Pass | Break | Continue + //Pass, Break, - Error + //Continue } #[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum Expr { - UnaryOp(UnaryOp, Box<Expr>), - BinOp(Box<Expr>, BinOp, Box<Expr>), - Compare(Box<Expr>, Vec<BinOp>, Vec<Expr>), - Call(Box<Expr>, Vec<Expr>), - Alias(String, Option<String>), - Attribute(Box<Expr>, String), - Name(String), +pub enum expr { + BoolOp(boolop, Vec<expr>), + BinOp(Box<expr>, operator, Box<expr>), + UnaryOp(unaryop, Box<expr>), + //Lambda(arguments, Box<expr>) + //IfExp(Box<expr>, Box<expr>, Box<expr>) + //Dict(Vec<expr>, Vec<expr>) + //Set(Vec<expr>) + //ListComp(Box<expr>, Vec<comprehension>), + //SetComp(Box<expr>, Vec<comprehension>) + //DictComp(Box<expr>, Box<expr>, Vec<comprehension>) + //GeneratorExp(Box<expr>, Vec<comprehension>) + + // the grammar constrains where yield expressions can occur + //Await(expr value) + //Yield(expr? value) + //YieldFrom(expr value) + + // need sequences for compare to distinguish between + // x < 4 < 3 and (x < 4) < 3 + Compare(Box<expr>, Vec<cmpop>, Vec<expr>), + + Call(Box<expr>, Vec<expr>, Vec<keyword>), + Num(String), + Str(String), + //Bytes(String), NameConstant(String), - Str(String), - Num(String), - List(Vec<Expr>), - Error + //Ellipsis, + + // the following expression can appear in assignment context + Attribute(Box<expr>, String, expr_context), + //Subscript(Box<expr>, slice, expr_context), + //Starred(Box<expr>, expr_context), + Name(String, expr_context), + List(Vec<expr>, expr_context), + //Tuple(Vec<expr>, expr_context), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum expr_context { + Load, + Store, + Del, + AugLoad, + AugStore, + Param +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum slice { + Slice(Option<expr>, Option<expr>, Option<expr>), + ExtSlice(Vec<slice>), + Index(expr) +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum boolop { + And, + Or } #[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum UnaryOp { +pub enum operator { + Add, + Sub, + Mult, + MatMult, + Div, + Mod, + Pow, + LShift, + RShift, + BitOr, + BitXor, + BitAnd, + FloorDiv +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum unaryop { + Invert, + Not, UAdd, - USub, - Error + USub +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum cmpop { + Eq, + NotEq, + Lt, + LtE, + Gt, + GtE, + Is, + IsNot, + In, + NotIn } #[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum BinOp { - BinAdd, - BinMult, - BinEq, - BinLt, - BinGt, - Sub, - Div, - Error +pub struct comprehension { + pub target: expr, + pub iter: expr, + pub ifs: Vec<expr> +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum excepthandler { + ExceptHandler(Option<expr>, Option<String>, Vec<stmt>) +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct arguments { + pub args: Vec<arg>, + pub vararg: Option<arg>, + pub kwonlyargs: Vec<arg>, + pub kw_defaults: Vec<expr>, + pub kwarg: Option<arg>, + pub defaults: Vec<expr> } + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct arg { + pub arg: String, + pub annotation: Option<expr> +} + +// keyword arguments supplied to call (NULL identifier for **kwargs) +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct keyword { + pub arg: Option<String>, + pub value: expr +} + +// import name with optional 'as' alias. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct alias { + pub name: String, + pub asname: Option<String> +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct withitem { + pub context_expr: expr, + pub optional_vars: Option<expr> +}