From d957816dede5f001de03ea90800cd83bb3929e5a Mon Sep 17 00:00:00 2001 From: ElnuDev Date: Sat, 20 May 2023 17:37:53 -0700 Subject: [PATCH] Convert to internal tokens --- demo/src/main.rs | 5 +++- src/lib.rs | 66 +++++++++++++++++++++++++++++------------------- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/demo/src/main.rs b/demo/src/main.rs index 4e2d100..8045c4e 100644 --- a/demo/src/main.rs +++ b/demo/src/main.rs @@ -1,5 +1,8 @@ use renrs; fn main() { - renrs::parse("demo.rpy"); + let tokens = renrs::parse_file("demo.rpy"); + for token in tokens { + println!("{:?}", token); + } } diff --git a/src/lib.rs b/src/lib.rs index 0f42168..0dfeb54 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,35 @@ use pest_derive::Parser; #[grammar = "rpy.pest"] struct RpyParser; -fn describe_token(pair: pest::iterators::Pair) { +#[derive(Debug)] +pub enum Token { + Keyword(String), + String(String), + Array(Vec), +} + +pub fn parse(script: &str) -> Vec> { + let file = RpyParser::parse(Rule::file, script) + .expect("unsuccessful parse") + .next().unwrap(); + let mut lines = Vec::new(); + for line in file.into_inner() { + let mut tokens = Vec::new(); + match line.as_rule() { + Rule::line => { + for token in line.into_inner() { + tokens.push(parse_token(token)); + } + }, + Rule::EOI => (), + _ => unreachable!(), + } + lines.push(tokens); + } + lines +} + +fn parse_token(pair: pest::iterators::Pair) -> Token { let token = pair.as_rule(); match token { Rule::token => {}, @@ -15,42 +43,28 @@ fn describe_token(pair: pest::iterators::Pair) { }; let contents = pair.into_inner().next().unwrap(); let contents_rule = contents.as_rule(); - let str = match contents_rule { + match contents_rule { Rule::string => { let data = contents.into_inner().next().unwrap(); - match data.as_rule() { + Token::String(match data.as_rule() { Rule::single_quote_string_data => data.as_str().replace("\\'", "'"), Rule::double_quote_string_data => data.as_str().replace("\\\"", "\""), _ => unreachable!(), - } + }) }, Rule::array => { - println!("array: Start array"); + let mut array = Vec::new(); for token in contents.into_inner() { - describe_token(token); + array.push(parse_token(token)); } - "End array".to_string() + Token::Array(array) } - _ => contents.as_str().to_owned(), - }; - println!("{:?}: {}", contents_rule, str); + Rule::keyword => Token::Keyword(contents.as_str().to_owned()), + __ => unreachable!(), + } } -pub fn parse(file_path: &str) { +pub fn parse_file(file_path: &str) -> Vec> { let unparsed_file = fs::read_to_string(file_path).expect("cannot find file"); - let file = RpyParser::parse(Rule::file, &unparsed_file) - .expect("unsuccessful parse") // unwrap the parse result - .next().unwrap(); // get and unwrap the `file` rule; never fails - for line in file.into_inner() { - match line.as_rule() { - Rule::line => { - for token in line.into_inner() { - describe_token(token); - } - println!() - }, - Rule::EOI => (), - _ => unreachable!(), - } - } + parse(&unparsed_file) }