jeff/dust
1
0
Fork 0
Code Issues

Reduce use of TokenKind in parser to only error cases

Browse Source
This commit is contained in:
Jeff 2024-10-12 06:27:19 -04:00
parent 30b2801a38
commit 02877d06d3
1 changed files with 91 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

177
dust-lang/src/parser.rs
View File

@ -123,8 +123,8 @@ impl<'src> Parser<'src> {
Ok(()) Ok(())
} }
fn allow(&mut self, allowed: TokenKind) -> Result<bool, ParseError> { fn allow(&mut self, allowed: Token) -> Result<bool, ParseError> {
if self.current_token.kind() == allowed { if self.current_token == allowed {
self.advance()?; self.advance()?;
Ok(true) Ok(true)
@ -133,12 +133,12 @@ impl<'src> Parser<'src> {
} }
} }
fn expect(&mut self, expected: TokenKind) -> Result<(), ParseError> { fn expect(&mut self, expected: Token) -> Result<(), ParseError> {
if self.current_token.kind() == expected { if self.current_token == expected {
self.advance() self.advance()
} else { } else {
Err(ParseError::ExpectedToken { Err(ParseError::ExpectedToken {
expected, expected: expected.kind(),
found: self.current_token.to_owned(), found: self.current_token.to_owned(),
position: self.current_position, position: self.current_position,
}) })
@ -289,7 +289,7 @@ impl<'src> Parser<'src> {
} }
fn parse_grouped(&mut self, _: Allowed) -> Result<(), ParseError> { fn parse_grouped(&mut self, _: Allowed) -> Result<(), ParseError> {
self.allow(TokenKind::LeftParenthesis)?; self.allow(Token::LeftParenthesis)?;
self.parse_statement( self.parse_statement(
Allowed { Allowed {
assignment: false, assignment: false,
@ -298,7 +298,7 @@ impl<'src> Parser<'src> {
}, },
Context::None, Context::None,
)?; )?;
self.expect(TokenKind::RightParenthesis)?; self.expect(Token::RightParenthesis)?;
self.parsed_expression = true; self.parsed_expression = true;
@ -417,7 +417,7 @@ impl<'src> Parser<'src> {
let operator = self.current_token; let operator = self.current_token;
let operator_position = self.current_position; let operator_position = self.current_position;
let rule = ParseRule::from(&operator.kind()); let rule = ParseRule::from(&operator);
if let TokenKind::PlusEqual if let TokenKind::PlusEqual
| TokenKind::MinusEqual | TokenKind::MinusEqual
@ -547,7 +547,7 @@ impl<'src> Parser<'src> {
let operator = self.current_token; let operator = self.current_token;
let operator_position = self.current_position; let operator_position = self.current_position;
let rule = ParseRule::from(&operator.kind()); let rule = ParseRule::from(&operator);
self.advance()?; self.advance()?;
self.parse( self.parse(
@ -618,11 +618,11 @@ impl<'src> Parser<'src> {
let operator = self.current_token; let operator = self.current_token;
let operator_position = self.current_position; let operator_position = self.current_position;
let rule = ParseRule::from(&operator.kind()); let rule = ParseRule::from(&operator);
let instruction = match operator.kind() { let instruction = match operator {
TokenKind::DoubleAmpersand => Instruction::test(left_instruction.a(), false), Token::DoubleAmpersand => Instruction::test(left_instruction.a(), false),
TokenKind::DoublePipe => Instruction::test(left_instruction.a(), true), Token::DoublePipe => Instruction::test(left_instruction.a(), true),
_ => { _ => {
return Err(ParseError::ExpectedTokenMultiple { return Err(ParseError::ExpectedTokenMultiple {
expected: &[TokenKind::DoubleAmpersand, TokenKind::DoublePipe], expected: &[TokenKind::DoubleAmpersand, TokenKind::DoublePipe],
@ -664,7 +664,7 @@ impl<'src> Parser<'src> {
let local_index = self.parse_identifier_from(token, start_position)?; let local_index = self.parse_identifier_from(token, start_position)?;
if self.allow(TokenKind::Equal)? { if self.allow(Token::Equal)? {
if !allowed.assignment { if !allowed.assignment {
return Err(ParseError::InvalidAssignmentTarget { return Err(ParseError::InvalidAssignmentTarget {
found: self.current_token.to_owned(), found: self.current_token.to_owned(),
@ -775,7 +775,7 @@ impl<'src> Parser<'src> {
self.advance()?; self.advance()?;
self.chunk.begin_scope(); self.chunk.begin_scope();
while !self.allow(TokenKind::RightCurlyBrace)? && !self.is_eof() { while !self.allow(Token::RightCurlyBrace)? && !self.is_eof() {
self.parse_statement(allowed, Context::None)?; self.parse_statement(allowed, Context::None)?;
} }
@ -791,7 +791,7 @@ impl<'src> Parser<'src> {
let start_register = self.current_register; let start_register = self.current_register;
while !self.allow(TokenKind::RightSquareBrace)? && !self.is_eof() { while !self.allow(Token::RightSquareBrace)? && !self.is_eof() {
let next_register = self.current_register; let next_register = self.current_register;
self.parse_statement( self.parse_statement(
@ -814,7 +814,7 @@ impl<'src> Parser<'src> {
); );
} }
self.allow(TokenKind::Comma)?; self.allow(Token::Comma)?;
} }
let end_register = self.current_register - 1; let end_register = self.current_register - 1;
@ -874,7 +874,7 @@ impl<'src> Parser<'src> {
self.emit_instruction(load_constant, load_constant_position); self.emit_instruction(load_constant, load_constant_position);
} }
if self.allow(TokenKind::Else)? { if self.allow(Token::Else)? {
if let Token::If = self.current_token { if let Token::If = self.current_token {
self.parse_if(allowed)?; self.parse_if(allowed)?;
} }
@ -955,7 +955,7 @@ impl<'src> Parser<'src> {
} }
} }
let has_semicolon = self.allow(TokenKind::Semicolon)?; let has_semicolon = self.allow(Token::Semicolon)?;
if !has_semicolon && self.parsed_expression && allowed.implicit_return { if !has_semicolon && self.parsed_expression && allowed.implicit_return {
self.emit_instruction(Instruction::r#return(true), self.current_position); self.emit_instruction(Instruction::r#return(true), self.current_position);
@ -1012,7 +1012,7 @@ impl<'src> Parser<'src> {
self.advance()?; self.advance()?;
let is_mutable = self.allow(TokenKind::Mut)?; let is_mutable = self.allow(Token::Mut)?;
let position = self.current_position; let position = self.current_position;
let identifier = if let Token::Identifier(text) = self.current_token { let identifier = if let Token::Identifier(text) = self.current_token {
self.advance()?; self.advance()?;
@ -1025,7 +1025,7 @@ impl<'src> Parser<'src> {
position, position,
}); });
}; };
let r#type = if self.allow(TokenKind::Colon)? { let r#type = if self.allow(Token::Colon)? {
let r#type = self.parse_type_from(self.current_token, self.current_position)?; let r#type = self.parse_type_from(self.current_token, self.current_position)?;
self.advance()?; self.advance()?;
@ -1036,7 +1036,7 @@ impl<'src> Parser<'src> {
}; };
let register = self.current_register; let register = self.current_register;
self.expect(TokenKind::Equal)?; self.expect(Token::Equal)?;
self.parse_statement( self.parse_statement(
Allowed { Allowed {
@ -1069,11 +1069,11 @@ impl<'src> Parser<'src> {
let function_start = self.current_position.0; let function_start = self.current_position.0;
let mut function_parser = Parser::new(self.lexer)?; let mut function_parser = Parser::new(self.lexer)?;
function_parser.expect(TokenKind::LeftParenthesis)?; function_parser.expect(Token::LeftParenthesis)?;
while function_parser.current_token != Token::RightParenthesis { while function_parser.current_token != Token::RightParenthesis {
let start = function_parser.current_position.0; let start = function_parser.current_position.0;
let is_mutable = function_parser.allow(TokenKind::Mut)?; let is_mutable = function_parser.allow(Token::Mut)?;
let parameter = if let Token::Identifier(text) = function_parser.current_token { let parameter = if let Token::Identifier(text) = function_parser.current_token {
function_parser.advance()?; function_parser.advance()?;
@ -1086,7 +1086,7 @@ impl<'src> Parser<'src> {
}); });
}; };
function_parser.expect(TokenKind::Colon)?; function_parser.expect(Token::Colon)?;
let r#type = function_parser.parse_type_from( let r#type = function_parser.parse_type_from(
function_parser.current_token, function_parser.current_token,
@ -1110,11 +1110,11 @@ impl<'src> Parser<'src> {
Span(start, end), Span(start, end),
)?; )?;
function_parser.increment_register()?; function_parser.increment_register()?;
function_parser.allow(TokenKind::Comma)?; function_parser.allow(Token::Comma)?;
} }
function_parser.advance()?; function_parser.advance()?;
function_parser.expect(TokenKind::LeftCurlyBrace)?; function_parser.expect(Token::LeftCurlyBrace)?;
while function_parser.current_token != Token::RightCurlyBrace { while function_parser.current_token != Token::RightCurlyBrace {
function_parser.parse_statement( function_parser.parse_statement(
@ -1146,7 +1146,7 @@ impl<'src> Parser<'src> {
} }
fn parse(&mut self, precedence: Precedence, allowed: Allowed) -> Result<(), ParseError> { fn parse(&mut self, precedence: Precedence, allowed: Allowed) -> Result<(), ParseError> {
if let Some(prefix_parser) = ParseRule::from(&self.current_token.kind()).prefix { if let Some(prefix_parser) = ParseRule::from(&self.current_token).prefix {
log::debug!( log::debug!(
"{} is {precedence} prefix", "{} is {precedence} prefix",
self.current_token.to_string().bold(), self.current_token.to_string().bold(),
@ -1155,7 +1155,7 @@ impl<'src> Parser<'src> {
prefix_parser(self, allowed)?; prefix_parser(self, allowed)?;
} }
let mut infix_rule = ParseRule::from(&self.current_token.kind()); let mut infix_rule = ParseRule::from(&self.current_token);
while precedence <= infix_rule.precedence { while precedence <= infix_rule.precedence {
if let Some(infix_parser) = infix_rule.infix { if let Some(infix_parser) = infix_rule.infix {
@ -1176,7 +1176,7 @@ impl<'src> Parser<'src> {
break; break;
} }
infix_rule = ParseRule::from(&self.current_token.kind()); infix_rule = ParseRule::from(&self.current_token);
} }
Ok(()) Ok(())
@ -1247,227 +1247,232 @@ struct ParseRule<'a> {
pub precedence: Precedence, pub precedence: Precedence,
} }
impl From<&TokenKind> for ParseRule<'_> { impl From<&Token<'_>> for ParseRule<'_> {
fn from(token_kind: &TokenKind) -> Self { fn from(token_kind: &Token) -> Self {
match token_kind { match token_kind {
TokenKind::Async => todo!(), Token::Async => todo!(),
TokenKind::Bang => ParseRule { Token::Bang => ParseRule {
prefix: Some(Parser::parse_unary), prefix: Some(Parser::parse_unary),
infix: None, infix: None,
precedence: Precedence::Unary, precedence: Precedence::Unary,
}, },
TokenKind::BangEqual => ParseRule { Token::BangEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_comparison_binary), infix: Some(Parser::parse_comparison_binary),
precedence: Precedence::Equality, precedence: Precedence::Equality,
}, },
TokenKind::Bool => todo!(), Token::Bool => todo!(),
TokenKind::Boolean => ParseRule { Token::Boolean(_) => ParseRule {
prefix: Some(Parser::parse_boolean), prefix: Some(Parser::parse_boolean),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Break => todo!(), Token::Break => todo!(),
TokenKind::Byte => ParseRule { Token::Byte(_) => ParseRule {
prefix: Some(Parser::parse_byte), prefix: Some(Parser::parse_byte),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Character => ParseRule { Token::Character(_) => ParseRule {
prefix: Some(Parser::parse_character), prefix: Some(Parser::parse_character),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Colon => todo!(), Token::Colon => todo!(),
TokenKind::Comma => ParseRule { Token::Comma => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Dot => todo!(), Token::Dot => todo!(),
TokenKind::DoubleAmpersand => ParseRule { Token::DoubleAmpersand => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_logical_binary), infix: Some(Parser::parse_logical_binary),
precedence: Precedence::LogicalAnd, precedence: Precedence::LogicalAnd,
}, },
TokenKind::DoubleEqual => ParseRule { Token::DoubleEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_comparison_binary), infix: Some(Parser::parse_comparison_binary),
precedence: Precedence::Equality, precedence: Precedence::Equality,
}, },
TokenKind::DoublePipe => ParseRule { Token::DoublePipe => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_logical_binary), infix: Some(Parser::parse_logical_binary),
precedence: Precedence::LogicalOr, precedence: Precedence::LogicalOr,
}, },
TokenKind::DoubleDot => todo!(), Token::DoubleDot => todo!(),
TokenKind::Eof => ParseRule { Token::Eof => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Equal => ParseRule { Token::Equal => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::Assignment, precedence: Precedence::Assignment,
}, },
TokenKind::Else => ParseRule { Token::Else => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Float => ParseRule { Token::Float(_) => ParseRule {
prefix: Some(Parser::parse_float), prefix: Some(Parser::parse_float),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::FloatKeyword => todo!(), Token::FloatKeyword => todo!(),
TokenKind::Fn => ParseRule { Token::Fn => ParseRule {
prefix: Some(Parser::parse_function), prefix: Some(Parser::parse_function),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Greater => ParseRule { Token::Greater => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_comparison_binary), infix: Some(Parser::parse_comparison_binary),
precedence: Precedence::Comparison, precedence: Precedence::Comparison,
}, },
TokenKind::GreaterEqual => ParseRule { Token::GreaterEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_comparison_binary), infix: Some(Parser::parse_comparison_binary),
precedence: Precedence::Comparison, precedence: Precedence::Comparison,
}, },
TokenKind::Identifier => ParseRule { Token::Identifier(_) => ParseRule {
prefix: Some(Parser::parse_variable), prefix: Some(Parser::parse_variable),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::If => ParseRule { Token::If => ParseRule {
prefix: Some(Parser::parse_if), prefix: Some(Parser::parse_if),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Int => todo!(), Token::Int => todo!(),
TokenKind::Integer => ParseRule { Token::Integer(_) => ParseRule {
prefix: Some(Parser::parse_integer), prefix: Some(Parser::parse_integer),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::LeftCurlyBrace => ParseRule { Token::LeftCurlyBrace => ParseRule {
prefix: Some(Parser::parse_block), prefix: Some(Parser::parse_block),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::LeftParenthesis => ParseRule { Token::LeftParenthesis => ParseRule {
prefix: Some(Parser::parse_grouped), prefix: Some(Parser::parse_grouped),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::LeftSquareBrace => ParseRule { Token::LeftSquareBrace => ParseRule {
prefix: Some(Parser::parse_list), prefix: Some(Parser::parse_list),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Less => ParseRule { Token::Less => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_comparison_binary), infix: Some(Parser::parse_comparison_binary),
precedence: Precedence::Comparison, precedence: Precedence::Comparison,
}, },
TokenKind::LessEqual => ParseRule { Token::LessEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_comparison_binary), infix: Some(Parser::parse_comparison_binary),
precedence: Precedence::Comparison, precedence: Precedence::Comparison,
}, },
TokenKind::Let => ParseRule { Token::Let => ParseRule {
prefix: Some(Parser::parse_let_statement), prefix: Some(Parser::parse_let_statement),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Loop => todo!(), Token::Loop => todo!(),
TokenKind::Map => todo!(), Token::Map => todo!(),
TokenKind::Minus => ParseRule { Token::Minus => ParseRule {
prefix: Some(Parser::parse_unary), prefix: Some(Parser::parse_unary),
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Term, precedence: Precedence::Term,
}, },
TokenKind::MinusEqual => ParseRule { Token::MinusEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Assignment, precedence: Precedence::Assignment,
}, },
TokenKind::Mut => ParseRule { Token::Mut => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Percent => ParseRule { Token::Percent => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Factor, precedence: Precedence::Factor,
}, },
TokenKind::Plus => ParseRule { Token::PercentEqual => ParseRule {
prefix: None,
infix: Some(Parser::parse_math_binary),
precedence: Precedence::Assignment,
},
Token::Plus => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Term, precedence: Precedence::Term,
}, },
TokenKind::PlusEqual => ParseRule { Token::PlusEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Assignment, precedence: Precedence::Assignment,
}, },
TokenKind::Return => ParseRule { Token::Return => ParseRule {
prefix: Some(Parser::parse_return), prefix: Some(Parser::parse_return),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::RightCurlyBrace => ParseRule { Token::RightCurlyBrace => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::RightParenthesis => ParseRule { Token::RightParenthesis => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::RightSquareBrace => ParseRule { Token::RightSquareBrace => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Semicolon => ParseRule { Token::Semicolon => ParseRule {
prefix: None, prefix: None,
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Slash => ParseRule { Token::Slash => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Factor, precedence: Precedence::Factor,
}, },
TokenKind::SlashEqual => ParseRule { Token::SlashEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Assignment, precedence: Precedence::Assignment,
}, },
TokenKind::Star => ParseRule { Token::Star => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Factor, precedence: Precedence::Factor,
}, },
TokenKind::StarEqual => ParseRule { Token::StarEqual => ParseRule {
prefix: None, prefix: None,
infix: Some(Parser::parse_math_binary), infix: Some(Parser::parse_math_binary),
precedence: Precedence::Assignment, precedence: Precedence::Assignment,
}, },
TokenKind::Str => todo!(), Token::Str => todo!(),
TokenKind::String => ParseRule { Token::String(_) => ParseRule {
prefix: Some(Parser::parse_string), prefix: Some(Parser::parse_string),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,
}, },
TokenKind::Struct => todo!(), Token::Struct => todo!(),
TokenKind::While => ParseRule { Token::While => ParseRule {
prefix: Some(Parser::parse_while), prefix: Some(Parser::parse_while),
infix: None, infix: None,
precedence: Precedence::None, precedence: Precedence::None,