ElnuDev
/
renrs
generated from ElnuDev/rust-project
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simplify parsing

Browse Source
main
Elnu 2 years ago
parent 6707b97ef5
commit c0c8153cbe
2 changed files with 76 additions and 119 deletions
Show Stats Download Patch File Download Diff File

183
renrs/src/lib.rs
Unescape View File

@ -32,19 +32,15 @@ impl Token {
use Token::*; use Token::*;
// Parsing types
type Line = Vec<Token>;
// Indented command block // Indented command block
#[derive(Debug)] #[derive(Debug)]
struct Block<T> { struct CommandBlock {
elements: Vec<BlockElement<T>>, elements: Vec<BlockElement>,
next: Option<usize>, next: Option<usize>,
} }
impl<T> Block<T> { impl CommandBlock {
fn next(&mut self) -> Option<&T> { fn next(&mut self) -> Option<&Command> {
let mut next = match self.next { let mut next = match self.next {
Some(next) => next, Some(next) => next,
None => return None, None => return None,
@ -80,7 +76,7 @@ impl<T> Block<T> {
} }
} }
impl<T> Default for Block<T> { impl Default for CommandBlock {
fn default() -> Self { fn default() -> Self {
Self { Self {
elements: Vec::new(), elements: Vec::new(),
@ -90,13 +86,11 @@ impl<T> Default for Block<T> {
} }
#[derive(Debug)] #[derive(Debug)]
enum BlockElement<T> { enum BlockElement {
Command(T), Command(Command),
Block(Block<T>), Block(CommandBlock),
} }
type LineBlock = Block<Line>;
type CommandBlock = Block<Command>;
type Script = CommandBlock; type Script = CommandBlock;
// Parsed script commands // Parsed script commands
@ -123,116 +117,44 @@ pub enum Event {
Say { name: Option<String>, text: String }, Say { name: Option<String>, text: String },
} }
// ========================================== // ======
// Step 1 parsing // Parser
// converting from pest pairs to Token blocks // ======
// ==========================================
// Tokenize raw script string
fn parse(script: &str) -> LineBlock {
let file = RpyParser::parse(Rule::file, script)
.expect("unsuccessful parse")
.next()
.unwrap();
parse_block(file)
}
type Pair<'a> = pest::iterators::Pair<'a, Rule>; type Pair<'a> = pest::iterators::Pair<'a, Rule>;
// Tokenize block // Read file into commands
fn parse_block(pair: Pair) -> LineBlock { fn parse_file(file_path: &PathBuf) -> CommandBlock {
let mut block = LineBlock::default(); let unparsed_file = fs::read_to_string(file_path).expect("cannot find file");
for element in pair.into_inner() { parse(&unparsed_file)
block.elements.push(match element.as_rule() {
Rule::block => BlockElement::Block(parse_block(element)),
Rule::line => {
let line = parse_line(element);
// TODO: For some reason a blank final line is always parsed
if line.len() == 0 {
continue;
}
BlockElement::Command(line)
},
Rule::EOI => break, // end
_ => unreachable!(),
});
}
block
}
// Tokenize line
fn parse_line(pair: Pair) -> Line {
let mut tokens = Vec::new();
for token in pair.into_inner() {
tokens.push(parse_token(token));
}
tokens
}
// Tokenize token
fn parse_token(pair: Pair) -> Token {
let token = pair.as_rule();
macro_rules! contents {
() => {
pair.into_inner().next().unwrap()
};
}
match token {
Rule::String => {
let contents = contents!();
Token::Str(match contents.as_rule() {
Rule::SingleQuoteStringData => contents.as_str().replace("\\'", "'"),
Rule::DoubleQuoteStringData => contents.as_str().replace("\\\"", "\""),
_ => unreachable!(),
})
}
Rule::Array => {
let contents = contents!();
let mut array = Vec::new();
for token in contents.into_inner() {
array.push(parse_token(token));
}
Token::Array(array)
}
Rule::Boolean => Token::Boolean(match pair.as_str() {
"True" => true,
"False" => false,
_ => unreachable!(),
}),
Rule::Number => Token::Number(pair.as_str().parse().unwrap()),
Rule::Keyword => Token::Keyword(pair.as_str().to_owned()),
__ => unreachable!(),
}
} }
fn parse(script: &str) -> CommandBlock {
// ============================================== let file = RpyParser::parse(Rule::File, script)
// Step 2 reading .expect("unsuccessful parse")
// converting from Token blocks to Command blocks .next()
// ============================================== .unwrap();
parse_block(file)
// Read file into commands
fn read_file(file_path: &PathBuf) -> CommandBlock {
let line_block = {
let unparsed_file = fs::read_to_string(file_path).expect("cannot find file");
parse(&unparsed_file)
};
read_block(&line_block)
} }
// Read line block into command block fn parse_block(block: Pair) -> CommandBlock {
fn read_block(block: &LineBlock) -> CommandBlock {
CommandBlock { CommandBlock {
elements: block.elements.iter().map(|element| match element { elements: block.into_inner().filter_map(|pair| match pair.as_rule() {
BlockElement::Command(line) => BlockElement::Command(read_command(&line)), Rule::Line => Some(BlockElement::Command(parse_command(pair))),
BlockElement::Block(block) => BlockElement::Block(read_block(&block)), Rule::Block => Some(BlockElement::Block(parse_block(pair))),
Rule::EOI => None, // end
_ => unreachable!(),
}).collect(), }).collect(),
..Default::default() ..Default::default()
} }
} }
// Read token array to command fn parse_command(pair: Pair) -> Command {
fn read_command(line: &Line) -> Command { use Token::*;
let line: Vec<Token> = pair
.into_inner()
.map(|pair| parse_token(pair))
.collect();
macro_rules! unknown { macro_rules! unknown {
() => { () => {
panic!("Unknown command {}", describe_line(&line)) panic!("Unknown command {}", describe_line(&line))
@ -260,7 +182,7 @@ fn read_command(line: &Line) -> Command {
// Line description e.g. [String, Keyword, Array] // Line description e.g. [String, Keyword, Array]
// Used in parse_command as feedback for invalid commands // Used in parse_command as feedback for invalid commands
fn describe_line(line: &Line) -> String { fn describe_line(line: &[Token]) -> String {
let mut description = "[".to_owned(); let mut description = "[".to_owned();
let mut iter = line.iter(); let mut iter = line.iter();
description.push_str(&format!("{}", iter.next().unwrap().print())); description.push_str(&format!("{}", iter.next().unwrap().print()));
@ -271,6 +193,41 @@ fn describe_line(line: &Line) -> String {
description description
} }
fn parse_token(pair: Pair) -> Token {
let token = pair.as_rule();
macro_rules! contents {
() => {
pair.into_inner().next().unwrap()
};
}
match token {
Rule::String => {
let contents = contents!();
Token::Str(match contents.as_rule() {
Rule::SingleQuoteStringData => contents.as_str().replace("\\'", "'"),
Rule::DoubleQuoteStringData => contents.as_str().replace("\\\"", "\""),
_ => unreachable!(),
})
}
Rule::Array => {
let contents = contents!();
let mut array = Vec::new();
for token in contents.into_inner() {
array.push(parse_token(token));
}
Token::Array(array)
}
Rule::Boolean => Token::Boolean(match pair.as_str() {
"True" => true,
"False" => false,
_ => unreachable!(),
}),
Rule::Number => Token::Number(pair.as_str().parse().unwrap()),
Rule::Keyword => Token::Keyword(pair.as_str().to_owned()),
__ => unreachable!(),
}
}
// ===== // =====
// State // State
// ===== // =====
@ -282,7 +239,7 @@ pub struct State {
impl State { impl State {
pub fn from_file(file: PathBuf) -> State { pub fn from_file(file: PathBuf) -> State {
State { State {
script: read_file(&file), script: parse_file(&file),
} }
} }

12
renrs/src/rpy.pest
Unescape View File

@ -5,7 +5,7 @@ char = { !NEWLINE ~ ANY }
// http://pest.rs/book/grammars/syntax.html#atomic // http://pest.rs/book/grammars/syntax.html#atomic
inner = @{ char* } inner = @{ char* }
Token = { String | Array | Boolean | Number | Keyword } token = { String | Array | Boolean | Number | Keyword }
// KEYWORDS // KEYWORDS
// has to be atomic for no implicit separate (spaces) // has to be atomic for no implicit separate (spaces)
@ -28,7 +28,7 @@ String = ${
// Array // Array
Array = ${ Array = ${
"[" ~ "]" "[" ~ "]"
| "[" ~ whitespace* ~ NEWLINE* ~ whitespace* ~ Token ~ ("," ~ whitespace* ~ NEWLINE* ~ whitespace* ~ Token)* ~ NEWLINE* ~ "]" | "[" ~ whitespace* ~ NEWLINE* ~ whitespace* ~ token ~ ("," ~ whitespace* ~ NEWLINE* ~ whitespace* ~ token)* ~ NEWLINE* ~ "]"
} }
// BOOLEAN // BOOLEAN
@ -48,11 +48,11 @@ COMMENT = _{ "#" ~ char* }
Colon = { ":" } Colon = { ":" }
// lines are comprised of a statement // lines are comprised of a statement
line = @{ (Token ~ whitespace+)* ~ Token ~ Colon? } Line = @{ (token ~ whitespace+)* ~ token ~ Colon? }
file = { SOI ~ NEWLINE* ~ block_content* ~ NEWLINE* ~ EOI } File = { SOI ~ NEWLINE* ~ block_content* ~ NEWLINE* ~ EOI }
block = { Block = {
// The first line in the block // The first line in the block
PEEK_ALL ~ PUSH(" "+ | "\t"+) ~ block_content ~ PEEK_ALL ~ PUSH(" "+ | "\t"+) ~ block_content ~
// Subsequent lines in the block // Subsequent lines in the block
@ -66,5 +66,5 @@ block = {
whitespace = _{ " " } whitespace = _{ " " }
block_content = _{ block_content = _{
line ~ (whitespace+ ~ line)* ~ (NEWLINE | EOI) ~ block* Line ~ (whitespace+ ~ Line)* ~ (NEWLINE | EOI) ~ Block*
} }
Write Preview
Loading…
Cancel
Save