From d957816dede5f001de03ea90800cd83bb3929e5a Mon Sep 17 00:00:00 2001
From: ElnuDev <elnu@elnu.com>
Date: Sat, 20 May 2023 17:37:53 -0700
Subject: [PATCH] Convert to internal tokens

---
 demo/src/main.rs |  5 +++-
 src/lib.rs       | 66 +++++++++++++++++++++++++++++-------------------
 2 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/demo/src/main.rs b/demo/src/main.rs
index 4e2d100..8045c4e 100644
--- a/demo/src/main.rs
+++ b/demo/src/main.rs
@@ -1,5 +1,8 @@
 use renrs;
 
 fn main() {
-    renrs::parse("demo.rpy");
+    let tokens = renrs::parse_file("demo.rpy");
+    for token in tokens {
+        println!("{:?}", token);
+    }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 0f42168..0dfeb54 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,7 +7,35 @@ use pest_derive::Parser;
 #[grammar = "rpy.pest"]
 struct RpyParser;
 
-fn describe_token(pair: pest::iterators::Pair<Rule>) {
+#[derive(Debug)]
+pub enum Token {
+    Keyword(String),
+    String(String),
+    Array(Vec<Token>),
+}
+
+pub fn parse(script: &str) -> Vec<Vec<Token>> {
+    let file = RpyParser::parse(Rule::file, script)
+        .expect("unsuccessful parse")
+        .next().unwrap();
+    let mut lines = Vec::new();
+    for line in file.into_inner() {
+        let mut tokens = Vec::new();
+        match line.as_rule() {
+            Rule::line => {
+                for token in line.into_inner() {
+                    tokens.push(parse_token(token));
+                }
+            },
+            Rule::EOI => (),
+            _ => unreachable!(),
+        }
+        lines.push(tokens);
+    } 
+    lines
+}
+
+fn parse_token(pair: pest::iterators::Pair<Rule>) -> Token {
     let token = pair.as_rule();
     match token {
         Rule::token => {},
@@ -15,42 +43,28 @@ fn describe_token(pair: pest::iterators::Pair<Rule>) {
     };
     let contents = pair.into_inner().next().unwrap();
     let contents_rule = contents.as_rule();
-    let str = match contents_rule {
+    match contents_rule {
         Rule::string => {
             let data = contents.into_inner().next().unwrap();
-            match data.as_rule() {
+            Token::String(match data.as_rule() {
                 Rule::single_quote_string_data => data.as_str().replace("\\'", "'"),
                 Rule::double_quote_string_data => data.as_str().replace("\\\"", "\""),
                 _ => unreachable!(),
-            }
+            })
         },
         Rule::array => {
-            println!("array: Start array");
+            let mut array = Vec::new();
             for token in contents.into_inner() {
-                describe_token(token);
+                array.push(parse_token(token));
             }
-            "End array".to_string()
+            Token::Array(array)
         }
-        _ => contents.as_str().to_owned(),
-    };
-    println!("{:?}: {}", contents_rule, str);
+        Rule::keyword => Token::Keyword(contents.as_str().to_owned()),
+        __ => unreachable!(),
+    }
 }
 
-pub fn parse(file_path: &str) {
+pub fn parse_file(file_path: &str) -> Vec<Vec<Token>> {
     let unparsed_file = fs::read_to_string(file_path).expect("cannot find file");
-    let file = RpyParser::parse(Rule::file, &unparsed_file)
-        .expect("unsuccessful parse") // unwrap the parse result
-        .next().unwrap(); // get and unwrap the `file` rule; never fails
-    for line in file.into_inner() {
-        match line.as_rule() {
-            Rule::line => {
-                for token in line.into_inner() {
-                    describe_token(token);
-                }
-                println!()
-            },
-            Rule::EOI => (),
-            _ => unreachable!(),
-        }
-    }
+    parse(&unparsed_file)
 }