From 2e51112b4b71192f4f47007b1dae9006e8d08b64 Mon Sep 17 00:00:00 2001 From: Jeff Date: Thu, 7 Nov 2024 18:48:53 -0500 Subject: [PATCH] Continue overhauling the lexer --- dust-lang/src/compiler.rs | 30 ++- dust-lang/src/formatter.rs | 4 +- dust-lang/src/lexer.rs | 363 +++++++++++++++++++++++++++++++++---- dust-lang/src/token.rs | 88 ++++----- 4 files changed, 389 insertions(+), 96 deletions(-) diff --git a/dust-lang/src/compiler.rs b/dust-lang/src/compiler.rs index d2635c2..741b857 100644 --- a/dust-lang/src/compiler.rs +++ b/dust-lang/src/compiler.rs @@ -886,7 +886,7 @@ impl<'src> Compiler<'src> { self.advance()?; self.chunk.begin_scope(); - while !self.allow(Token::RightCurlyBrace)? && !self.is_eof() { + while !self.allow(Token::RightBrace)? && !self.is_eof() { self.parse(Precedence::None)?; } @@ -902,7 +902,7 @@ impl<'src> Compiler<'src> { let start_register = self.next_register(); - while !self.allow(Token::RightSquareBrace)? && !self.is_eof() { + while !self.allow(Token::RightBracket)? && !self.is_eof() { let expected_register = self.next_register(); self.parse_expression()?; @@ -953,11 +953,11 @@ impl<'src> Compiler<'src> { let if_block_start = self.chunk.len(); let if_block_start_position = self.current_position; - if let Token::LeftCurlyBrace = self.current_token { + if let Token::LeftBrace = self.current_token { self.parse_block()?; } else { return Err(CompileError::ExpectedToken { - expected: TokenKind::LeftCurlyBrace, + expected: TokenKind::LeftBrace, found: self.current_token.to_owned(), position: self.current_position, }); @@ -985,11 +985,11 @@ impl<'src> Compiler<'src> { if let Token::Else = self.current_token { self.advance()?; - if let Token::LeftCurlyBrace = self.current_token { + if let Token::LeftBrace = self.current_token { self.parse_block()?; } else { return Err(CompileError::ExpectedTokenMultiple { - expected: &[TokenKind::If, TokenKind::LeftCurlyBrace], + expected: &[TokenKind::If, TokenKind::LeftBrace], found: self.current_token.to_owned(), position: self.current_position, }); @@ -1147,7 +1147,7 @@ impl<'src> Compiler<'src> { loop { self.parse(Precedence::None)?; - if self.is_eof() || self.allow(Token::RightCurlyBrace)? { + if self.is_eof() || self.allow(Token::RightBrace)? { self.parse_implicit_return()?; break; @@ -1179,10 +1179,8 @@ impl<'src> Compiler<'src> { self.advance()?; - let has_return_value = if matches!( - self.current_token, - Token::Semicolon | Token::RightCurlyBrace - ) { + let has_return_value = if matches!(self.current_token, Token::Semicolon | Token::RightBrace) + { false } else { self.parse_expression()?; @@ -1335,7 +1333,7 @@ impl<'src> Compiler<'src> { None }; - function_compiler.expect(Token::LeftCurlyBrace)?; + function_compiler.expect(Token::LeftBrace)?; function_compiler.parse_top_level()?; self.previous_token = function_compiler.previous_token; @@ -1673,7 +1671,7 @@ impl From<&Token<'_>> for ParseRule<'_> { infix: None, precedence: Precedence::None, }, - Token::LeftCurlyBrace => ParseRule { + Token::LeftBrace => ParseRule { prefix: Some(Compiler::parse_block), infix: None, precedence: Precedence::None, @@ -1683,7 +1681,7 @@ impl From<&Token<'_>> for ParseRule<'_> { infix: Some(Compiler::parse_call), precedence: Precedence::Call, }, - Token::LeftSquareBrace => ParseRule { + Token::LeftBracket => ParseRule { prefix: Some(Compiler::parse_list), infix: None, precedence: Precedence::None, @@ -1745,7 +1743,7 @@ impl From<&Token<'_>> for ParseRule<'_> { infix: None, precedence: Precedence::None, }, - Token::RightCurlyBrace => ParseRule { + Token::RightBrace => ParseRule { prefix: None, infix: None, precedence: Precedence::None, @@ -1755,7 +1753,7 @@ impl From<&Token<'_>> for ParseRule<'_> { infix: None, precedence: Precedence::None, }, - Token::RightSquareBrace => ParseRule { + Token::RightBracket => ParseRule { prefix: None, infix: None, precedence: Precedence::None, diff --git a/dust-lang/src/formatter.rs b/dust-lang/src/formatter.rs index 5a68747..1bb5692 100644 --- a/dust-lang/src/formatter.rs +++ b/dust-lang/src/formatter.rs @@ -101,13 +101,13 @@ impl<'src> Formatter<'src> { String(string) => { self.push_colored(string.magenta()); } - LeftCurlyBrace => { + LeftBrace => { self.next_line.push_str(self.current_token.as_str()); self.commit_line(LineKind::OpenBlock); self.indent += 1; } - RightCurlyBrace => { + RightBrace => { self.commit_line(LineKind::CloseBlock); self.next_line.push_str(self.current_token.as_str()); diff --git a/dust-lang/src/lexer.rs b/dust-lang/src/lexer.rs index 3967827..a5470bd 100644 --- a/dust-lang/src/lexer.rs +++ b/dust-lang/src/lexer.rs @@ -41,11 +41,10 @@ pub fn lex<'tokens, 'src: 'tokens>( error: CompileError::Lex(error), source, })?; - let is_eof = matches!(token, Token::Eof); tokens.push((token, span)); - if is_eof { + if lexer.is_eof() { break; } } @@ -60,6 +59,7 @@ pub fn lex<'tokens, 'src: 'tokens>( pub struct Lexer<'src> { source: &'src str, position: usize, + is_eof: bool, } impl<'src> Lexer<'src> { @@ -68,6 +68,7 @@ impl<'src> Lexer<'src> { Lexer { source, position: 0, + is_eof: false, } } @@ -76,7 +77,7 @@ impl<'src> Lexer<'src> { } pub fn is_eof(&self) -> bool { - self.position >= self.source.len() + self.is_eof } pub fn skip_to(&mut self, position: usize) { @@ -92,21 +93,14 @@ impl<'src> Lexer<'src> { lexer(self)? } else { + self.is_eof = true; + (Token::Eof, Span(self.position, self.position)) }; Ok((token, span)) } - /// Peek at the next token without consuming the source. - pub fn peek_token(&mut self) -> Result<(Token<'src>, Span), LexError> { - let token = self.next_token()?; - - self.position -= token.0.len(); - - Ok(token) - } - /// Progress to the next character. fn next_char(&mut self) -> Option { if let Some(c) = self.source[self.position..].chars().next() { @@ -261,7 +255,7 @@ impl<'src> Lexer<'src> { } /// Lex an identifier token. - fn lex_alphanumeric(&mut self) -> Result<(Token<'src>, Span), LexError> { + fn lex_keyword_or_identifier(&mut self) -> Result<(Token<'src>, Span), LexError> { let start_pos = self.position; while let Some(c) = self.peek_char() { @@ -318,6 +312,247 @@ impl<'src> Lexer<'src> { Ok((Token::String(text), Span(start_pos, self.position))) } + + fn lex_char(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_position = self.position; + + self.next_char(); + + let char = self.source[self.position..].chars().next().unwrap(); + + self.next_char(); + + if self.peek_char() == Some('\'') { + self.next_char(); + } else { + return Err(LexError::ExpectedCharacter { + expected: '\'', + actual: self.peek_char().unwrap_or('\0'), + position: self.position, + }); + } + + let end_position = self.position; + + Ok((Token::Character(char), Span(start_position, end_position))) + } + + fn lex_plus(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::PlusEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Plus, Span(start_pos, self.position))) + } + } + + fn lex_minus(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::MinusEqual, Span(start_pos, self.position))) + } else if self.peek_chars(8) == "Infinity" { + self.position += 8; + + Ok((Token::Float("Infinity"), Span(start_pos, self.position))) + } else { + Ok((Token::Minus, Span(start_pos, self.position))) + } + } + + fn lex_star(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::StarEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Star, Span(start_pos, self.position))) + } + } + + fn lex_slash(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::SlashEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Slash, Span(start_pos, self.position))) + } + } + + fn lex_percent(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::PercentEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Percent, Span(start_pos, self.position))) + } + } + + fn lex_unexpected(&mut self) -> Result<(Token<'src>, Span), LexError> { + Err(LexError::UnexpectedCharacter { + actual: self.peek_char().unwrap_or('\0'), + position: self.position, + }) + } + + fn lex_exclamation_mark(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::BangEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Bang, Span(start_pos, self.position))) + } + } + + fn lex_equal(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::DoubleEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Equal, Span(start_pos, self.position))) + } + } + + fn lex_less_than(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::LessEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Less, Span(start_pos, self.position))) + } + } + + fn lex_greater_than(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + if let Some('=') = self.peek_char() { + self.next_char(); + + Ok((Token::GreaterEqual, Span(start_pos, self.position))) + } else { + Ok((Token::Greater, Span(start_pos, self.position))) + } + } + + fn lex_ampersand(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + let peek_char = self.peek_char(); + + if let Some('&') = peek_char { + self.next_char(); + + Ok((Token::DoubleAmpersand, Span(start_pos, self.position))) + } else if peek_char.is_none() { + Err(LexError::UnexpectedEndOfFile { + position: self.position, + }) + } else { + Err(LexError::ExpectedCharacter { + expected: '&', + actual: self.peek_char().unwrap(), + position: self.position, + }) + } + } + + fn lex_pipe(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + let peek_char = self.peek_char(); + + if let Some('|') = peek_char { + self.next_char(); + + Ok((Token::DoublePipe, Span(start_pos, self.position))) + } else if peek_char.is_none() { + Err(LexError::UnexpectedEndOfFile { + position: self.position, + }) + } else { + Err(LexError::ExpectedCharacter { + expected: '&', + actual: self.peek_char().unwrap(), + position: self.position, + }) + } + } + + fn lex_left_parenthesis(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + Ok((Token::LeftParenthesis, Span(start_pos, self.position))) + } + + fn lex_right_parenthesis(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + Ok((Token::RightParenthesis, Span(start_pos, self.position))) + } + + fn lex_left_bracket(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + Ok((Token::LeftBracket, Span(start_pos, self.position))) + } + + fn lex_right_bracket(&mut self) -> Result<(Token<'src>, Span), LexError> { + let start_pos = self.position; + + self.next_char(); + + Ok((Token::RightBracket, Span(start_pos, self.position))) + } } type LexerFn<'src> = fn(&mut Lexer<'src>) -> Result<(Token<'src>, Span), LexError>; @@ -332,7 +567,63 @@ impl<'src> From<&char> for LexRule<'src> { '0'..='9' => LexRule { lexer: Lexer::lex_numeric, }, - _ => panic!("Invalid character"), + 'Z'..='a' => LexRule { + lexer: Lexer::lex_keyword_or_identifier, + }, + '"' => LexRule { + lexer: Lexer::lex_string, + }, + '\'' => LexRule { + lexer: Lexer::lex_char, + }, + '+' => LexRule { + lexer: Lexer::lex_plus, + }, + '-' => LexRule { + lexer: Lexer::lex_minus, + }, + '*' => LexRule { + lexer: Lexer::lex_star, + }, + '/' => LexRule { + lexer: Lexer::lex_slash, + }, + '%' => LexRule { + lexer: Lexer::lex_percent, + }, + '!' => LexRule { + lexer: Lexer::lex_exclamation_mark, + }, + '=' => LexRule { + lexer: Lexer::lex_equal, + }, + '<' => LexRule { + lexer: Lexer::lex_less_than, + }, + '>' => LexRule { + lexer: Lexer::lex_greater_than, + }, + '&' => LexRule { + lexer: Lexer::lex_ampersand, + }, + '|' => LexRule { + lexer: Lexer::lex_pipe, + }, + '(' => LexRule { + lexer: Lexer::lex_left_parenthesis, + }, + ')' => LexRule { + lexer: Lexer::lex_right_parenthesis, + }, + '[' => LexRule { + lexer: Lexer::lex_left_bracket, + }, + ']' => LexRule { + lexer: Lexer::lex_right_bracket, + }, + _ => LexRule { + lexer: Lexer::lex_unexpected, + }, } } } @@ -464,7 +755,7 @@ mod tests { lex(input), Ok(vec![ (Token::Map, Span(0, 3)), - (Token::LeftCurlyBrace, Span(4, 5)), + (Token::LeftBrace, Span(4, 5)), (Token::Identifier("x"), Span(6, 7)), (Token::Equal, Span(8, 9)), (Token::String("1"), Span(10, 13)), @@ -476,7 +767,7 @@ mod tests { (Token::Identifier("z"), Span(22, 23)), (Token::Equal, Span(24, 25)), (Token::Float("3.0"), Span(26, 29)), - (Token::RightCurlyBrace, Span(30, 31)), + (Token::RightBrace, Span(30, 31)), (Token::Eof, Span(31, 31)), ]) ); @@ -540,7 +831,7 @@ mod tests { Ok(vec![ (Token::Struct, Span(0, 6)), (Token::Identifier("FooBar"), Span(7, 13)), - (Token::LeftCurlyBrace, Span(14, 15)), + (Token::LeftBrace, Span(14, 15)), (Token::Identifier("foo"), Span(16, 19)), (Token::Colon, Span(19, 20)), (Token::Int, Span(21, 24)), @@ -548,7 +839,7 @@ mod tests { (Token::Identifier("bar"), Span(26, 29)), (Token::Colon, Span(29, 30)), (Token::FloatKeyword, Span(31, 36)), - (Token::RightCurlyBrace, Span(37, 38)), + (Token::RightBrace, Span(37, 38)), (Token::Eof, Span(38, 38)) ]) ); @@ -561,16 +852,16 @@ mod tests { assert_eq!( lex(input), Ok(vec![ - (Token::LeftSquareBrace, Span(0, 1)), + (Token::LeftBracket, Span(0, 1)), (Token::Integer("1"), Span(1, 2)), (Token::Comma, Span(2, 3)), (Token::Integer("2"), Span(4, 5)), (Token::Comma, Span(5, 6)), (Token::Integer("3"), Span(7, 8)), - (Token::RightSquareBrace, Span(8, 9)), - (Token::LeftSquareBrace, Span(9, 10)), + (Token::RightBracket, Span(8, 9)), + (Token::LeftBracket, Span(9, 10)), (Token::Integer("1"), Span(10, 11)), - (Token::RightSquareBrace, Span(11, 12)), + (Token::RightBracket, Span(11, 12)), (Token::Eof, Span(12, 12)), ]) ) @@ -583,13 +874,13 @@ mod tests { assert_eq!( lex(input), Ok(vec![ - (Token::LeftSquareBrace, Span(0, 1)), + (Token::LeftBracket, Span(0, 1)), (Token::Integer("1"), Span(1, 2)), (Token::Comma, Span(2, 3)), (Token::Integer("2"), Span(4, 5)), (Token::Comma, Span(5, 6)), (Token::Integer("3"), Span(7, 8)), - (Token::RightSquareBrace, Span(8, 9)), + (Token::RightBracket, Span(8, 9)), (Token::Eof, Span(9, 9)), ]) ) @@ -602,7 +893,7 @@ mod tests { assert_eq!( lex(input), Ok(vec![ - (Token::LeftCurlyBrace, Span(0, 1)), + (Token::LeftBrace, Span(0, 1)), (Token::Identifier("a"), Span(1, 2)), (Token::Equal, Span(3, 4)), (Token::Integer("1"), Span(5, 6)), @@ -614,7 +905,7 @@ mod tests { (Token::Identifier("c"), Span(15, 16)), (Token::Equal, Span(17, 18)), (Token::Integer("3"), Span(19, 20)), - (Token::RightCurlyBrace, Span(20, 21)), + (Token::RightBrace, Span(20, 21)), (Token::Dot, Span(21, 22)), (Token::Identifier("c"), Span(22, 23)), (Token::Eof, Span(23, 23)), @@ -683,15 +974,15 @@ mod tests { (Token::Identifier("x"), Span(3, 4)), (Token::Less, Span(5, 6)), (Token::Integer("10"), Span(7, 9)), - (Token::LeftCurlyBrace, Span(10, 11)), + (Token::LeftBrace, Span(10, 11)), (Token::Identifier("x"), Span(12, 13)), (Token::Plus, Span(14, 15)), (Token::Integer("1"), Span(16, 17)), - (Token::RightCurlyBrace, Span(18, 19)), + (Token::RightBrace, Span(18, 19)), (Token::Else, Span(20, 24)), - (Token::LeftCurlyBrace, Span(25, 26)), + (Token::LeftBrace, Span(25, 26)), (Token::Identifier("x"), Span(27, 28)), - (Token::RightCurlyBrace, Span(29, 30)), + (Token::RightBrace, Span(29, 30)), (Token::Eof, Span(30, 30)), ]) ) @@ -708,11 +999,11 @@ mod tests { (Token::Identifier("x"), Span(6, 7)), (Token::Less, Span(8, 9)), (Token::Integer("10"), Span(10, 12)), - (Token::LeftCurlyBrace, Span(13, 14)), + (Token::LeftBrace, Span(13, 14)), (Token::Identifier("x"), Span(15, 16)), (Token::PlusEqual, Span(17, 19)), (Token::Integer("1"), Span(20, 21)), - (Token::RightCurlyBrace, Span(22, 23)), + (Token::RightBrace, Span(22, 23)), (Token::Eof, Span(23, 23)), ]) ) @@ -755,7 +1046,7 @@ mod tests { assert_eq!( lex(input), Ok(vec![ - (Token::LeftCurlyBrace, Span(0, 1)), + (Token::LeftBrace, Span(0, 1)), (Token::Identifier("x"), Span(2, 3)), (Token::Equal, Span(4, 5)), (Token::Integer("42"), Span(6, 8)), @@ -763,7 +1054,7 @@ mod tests { (Token::Identifier("y"), Span(10, 11)), (Token::Equal, Span(12, 13)), (Token::String("foobar"), Span(14, 22)), - (Token::RightCurlyBrace, Span(23, 24)), + (Token::RightBrace, Span(23, 24)), (Token::Eof, Span(24, 24)), ]) ) @@ -1104,8 +1395,8 @@ mod tests { assert_eq!( lex(input), Ok(vec![ - (Token::LeftSquareBrace, Span(0, 1)), - (Token::RightSquareBrace, Span(1, 2)), + (Token::LeftBracket, Span(0, 1)), + (Token::RightBracket, Span(1, 2)), (Token::Eof, Span(2, 2)), ]) ) diff --git a/dust-lang/src/token.rs b/dust-lang/src/token.rs index 4f1333b..c39ccac 100644 --- a/dust-lang/src/token.rs +++ b/dust-lang/src/token.rs @@ -9,15 +9,19 @@ use serde::{Deserialize, Serialize}; use crate::Span; pub fn output_token_list(tokens: &[(Token, Span)], writer: &mut W) { - const HEADER: [&str; 2] = ["TOKEN POSITION ", "------------ ----------"]; + const HEADER: [&str; 2] = [ + "TOKEN KIND POSITION ", + "------------ ---------- ----------", + ]; writeln!(writer, "{}", HEADER[0]).unwrap(); writeln!(writer, "{}", HEADER[1]).unwrap(); for (token, position) in tokens { + let kind = token.kind().to_string(); let token = token.to_string(); - writeln!(writer, "{token:<12} {position}").unwrap(); + writeln!(writer, "{token:<12} {kind:<10} {position}").unwrap(); } } @@ -90,9 +94,9 @@ define_tokens! { Equal, Greater, GreaterEqual, - LeftCurlyBrace, + LeftBrace, + LeftBracket, LeftParenthesis, - LeftSquareBrace, Less, LessEqual, Minus, @@ -101,9 +105,9 @@ define_tokens! { PercentEqual, Plus, PlusEqual, - RightCurlyBrace, + RightBrace, + RightBracket, RightParenthesis, - RightSquareBrace, Semicolon, Slash, SlashEqual, @@ -151,9 +155,9 @@ impl<'src> Token<'src> { Token::Equal => 1, Token::Greater => 1, Token::GreaterEqual => 2, - Token::LeftCurlyBrace => 1, + Token::LeftBrace => 1, Token::LeftParenthesis => 1, - Token::LeftSquareBrace => 1, + Token::LeftBracket => 1, Token::Less => 1, Token::LessEqual => 2, Token::Minus => 1, @@ -163,9 +167,9 @@ impl<'src> Token<'src> { Token::Plus => 1, Token::PlusEqual => 2, Token::Return => 6, - Token::RightCurlyBrace => 1, + Token::RightBrace => 1, Token::RightParenthesis => 1, - Token::RightSquareBrace => 1, + Token::RightBracket => 1, Token::Semicolon => 1, Token::Slash => 1, Token::SlashEqual => 2, @@ -212,9 +216,9 @@ impl<'src> Token<'src> { Token::Equal => "=", Token::Greater => ">", Token::GreaterEqual => ">=", - Token::LeftCurlyBrace => "{", + Token::LeftBrace => "{", Token::LeftParenthesis => "(", - Token::LeftSquareBrace => "[", + Token::LeftBracket => "[", Token::Less => "<", Token::LessEqual => "<=", Token::Minus => "-", @@ -224,9 +228,9 @@ impl<'src> Token<'src> { Token::Plus => "+", Token::PlusEqual => "+=", Token::Return => "return", - Token::RightCurlyBrace => "}", + Token::RightBrace => "}", Token::RightParenthesis => ")", - Token::RightSquareBrace => "]", + Token::RightBracket => "]", Token::Semicolon => ";", Token::Slash => "/", Token::SlashEqual => "/=", @@ -265,9 +269,9 @@ impl<'src> Token<'src> { Token::If => TokenOwned::If, Token::Int => TokenOwned::Int, Token::Integer(integer) => TokenOwned::Integer(integer.to_string()), - Token::LeftCurlyBrace => TokenOwned::LeftCurlyBrace, + Token::LeftBrace => TokenOwned::LeftCurlyBrace, Token::LeftParenthesis => TokenOwned::LeftParenthesis, - Token::LeftSquareBrace => TokenOwned::LeftSquareBrace, + Token::LeftBracket => TokenOwned::LeftSquareBrace, Token::Let => TokenOwned::Let, Token::Less => TokenOwned::Less, Token::LessEqual => TokenOwned::LessOrEqual, @@ -281,9 +285,9 @@ impl<'src> Token<'src> { Token::Plus => TokenOwned::Plus, Token::PlusEqual => TokenOwned::PlusEqual, Token::Return => TokenOwned::Return, - Token::RightCurlyBrace => TokenOwned::RightCurlyBrace, + Token::RightBrace => TokenOwned::RightCurlyBrace, Token::RightParenthesis => TokenOwned::RightParenthesis, - Token::RightSquareBrace => TokenOwned::RightSquareBrace, + Token::RightBracket => TokenOwned::RightSquareBrace, Token::Semicolon => TokenOwned::Semicolon, Token::Star => TokenOwned::Star, Token::StarEqual => TokenOwned::StarEqual, @@ -326,9 +330,9 @@ impl<'src> Token<'src> { Token::If => TokenKind::If, Token::Int => TokenKind::Int, Token::Integer(_) => TokenKind::Integer, - Token::LeftCurlyBrace => TokenKind::LeftCurlyBrace, + Token::LeftBrace => TokenKind::LeftBrace, Token::LeftParenthesis => TokenKind::LeftParenthesis, - Token::LeftSquareBrace => TokenKind::LeftSquareBrace, + Token::LeftBracket => TokenKind::LeftBracket, Token::Let => TokenKind::Let, Token::Less => TokenKind::Less, Token::LessEqual => TokenKind::LessEqual, @@ -342,9 +346,9 @@ impl<'src> Token<'src> { Token::Plus => TokenKind::Plus, Token::PlusEqual => TokenKind::PlusEqual, Token::Return => TokenKind::Return, - Token::RightCurlyBrace => TokenKind::RightCurlyBrace, + Token::RightBrace => TokenKind::RightBrace, Token::RightParenthesis => TokenKind::RightParenthesis, - Token::RightSquareBrace => TokenKind::RightSquareBrace, + Token::RightBracket => TokenKind::RightBracket, Token::Semicolon => TokenKind::Semicolon, Token::Star => TokenKind::Star, Token::StarEqual => TokenKind::StarEqual, @@ -381,9 +385,9 @@ impl<'src> Token<'src> { | Token::Equal | Token::Greater | Token::GreaterEqual - | Token::LeftCurlyBrace + | Token::LeftBrace | Token::LeftParenthesis - | Token::LeftSquareBrace + | Token::LeftBracket | Token::Less | Token::LessEqual | Token::Minus @@ -431,9 +435,9 @@ impl<'src> Display for Token<'src> { Token::If => write!(f, "if"), Token::Int => write!(f, "int"), Token::Integer(value) => write!(f, "{value}"), - Token::LeftCurlyBrace => write!(f, "{{"), + Token::LeftBrace => write!(f, "{{"), Token::LeftParenthesis => write!(f, "("), - Token::LeftSquareBrace => write!(f, "["), + Token::LeftBracket => write!(f, "["), Token::Let => write!(f, "let"), Token::Less => write!(f, "<"), Token::LessEqual => write!(f, "<="), @@ -447,9 +451,9 @@ impl<'src> Display for Token<'src> { Token::Plus => write!(f, "+"), Token::PlusEqual => write!(f, "+="), Token::Return => write!(f, "return"), - Token::RightCurlyBrace => write!(f, "}}"), + Token::RightBrace => write!(f, "}}"), Token::RightParenthesis => write!(f, ")"), - Token::RightSquareBrace => write!(f, "]"), + Token::RightBracket => write!(f, "]"), Token::Semicolon => write!(f, ";"), Token::Slash => write!(f, "/"), Token::SlashEqual => write!(f, "/="), @@ -564,9 +568,9 @@ impl Display for TokenOwned { TokenOwned::If => Token::If.fmt(f), TokenOwned::Int => Token::Int.fmt(f), TokenOwned::Integer(integer) => Token::Integer(integer).fmt(f), - TokenOwned::LeftCurlyBrace => Token::LeftCurlyBrace.fmt(f), + TokenOwned::LeftCurlyBrace => Token::LeftBrace.fmt(f), TokenOwned::LeftParenthesis => Token::LeftParenthesis.fmt(f), - TokenOwned::LeftSquareBrace => Token::LeftSquareBrace.fmt(f), + TokenOwned::LeftSquareBrace => Token::LeftBracket.fmt(f), TokenOwned::Let => Token::Let.fmt(f), TokenOwned::Less => Token::Less.fmt(f), TokenOwned::LessOrEqual => Token::LessEqual.fmt(f), @@ -580,9 +584,9 @@ impl Display for TokenOwned { TokenOwned::Plus => Token::Plus.fmt(f), TokenOwned::PlusEqual => Token::PlusEqual.fmt(f), TokenOwned::Return => Token::Return.fmt(f), - TokenOwned::RightCurlyBrace => Token::RightCurlyBrace.fmt(f), + TokenOwned::RightCurlyBrace => Token::RightBrace.fmt(f), TokenOwned::RightParenthesis => Token::RightParenthesis.fmt(f), - TokenOwned::RightSquareBrace => Token::RightSquareBrace.fmt(f), + TokenOwned::RightSquareBrace => Token::RightBracket.fmt(f), TokenOwned::Semicolon => Token::Semicolon.fmt(f), TokenOwned::Star => Token::Star.fmt(f), TokenOwned::StarEqual => Token::StarEqual.fmt(f), @@ -604,10 +608,10 @@ impl Display for TokenKind { TokenKind::Bang => Token::Bang.fmt(f), TokenKind::BangEqual => Token::BangEqual.fmt(f), TokenKind::Bool => Token::Bool.fmt(f), - TokenKind::Boolean => write!(f, "boolean value"), + TokenKind::Boolean => write!(f, "boolean"), TokenKind::Break => Token::Break.fmt(f), - TokenKind::Byte => write!(f, "byte value"), - TokenKind::Character => write!(f, "character value"), + TokenKind::Byte => write!(f, "byte"), + TokenKind::Character => write!(f, "character"), TokenKind::Colon => Token::Colon.fmt(f), TokenKind::Comma => Token::Comma.fmt(f), TokenKind::Dot => Token::Dot.fmt(f), @@ -618,7 +622,7 @@ impl Display for TokenKind { TokenKind::Else => Token::Else.fmt(f), TokenKind::Eof => Token::Eof.fmt(f), TokenKind::Equal => Token::Equal.fmt(f), - TokenKind::Float => write!(f, "float value"), + TokenKind::Float => write!(f, "float"), TokenKind::FloatKeyword => Token::FloatKeyword.fmt(f), TokenKind::Fn => Token::Fn.fmt(f), TokenKind::Greater => Token::Greater.fmt(f), @@ -626,10 +630,10 @@ impl Display for TokenKind { TokenKind::Identifier => write!(f, "identifier"), TokenKind::If => Token::If.fmt(f), TokenKind::Int => Token::Int.fmt(f), - TokenKind::Integer => write!(f, "integer value"), - TokenKind::LeftCurlyBrace => Token::LeftCurlyBrace.fmt(f), + TokenKind::Integer => write!(f, "integer"), + TokenKind::LeftBrace => Token::LeftBrace.fmt(f), TokenKind::LeftParenthesis => Token::LeftParenthesis.fmt(f), - TokenKind::LeftSquareBrace => Token::LeftSquareBrace.fmt(f), + TokenKind::LeftBracket => Token::LeftBracket.fmt(f), TokenKind::Let => Token::Let.fmt(f), TokenKind::Less => Token::Less.fmt(f), TokenKind::LessEqual => Token::LessEqual.fmt(f), @@ -643,16 +647,16 @@ impl Display for TokenKind { TokenKind::Plus => Token::Plus.fmt(f), TokenKind::PlusEqual => Token::PlusEqual.fmt(f), TokenKind::Return => Token::Return.fmt(f), - TokenKind::RightCurlyBrace => Token::RightCurlyBrace.fmt(f), + TokenKind::RightBrace => Token::RightBrace.fmt(f), TokenKind::RightParenthesis => Token::RightParenthesis.fmt(f), - TokenKind::RightSquareBrace => Token::RightSquareBrace.fmt(f), + TokenKind::RightBracket => Token::RightBracket.fmt(f), TokenKind::Semicolon => Token::Semicolon.fmt(f), TokenKind::Star => Token::Star.fmt(f), TokenKind::StarEqual => Token::StarEqual.fmt(f), TokenKind::Str => Token::Str.fmt(f), TokenKind::Slash => Token::Slash.fmt(f), TokenKind::SlashEqual => Token::SlashEqual.fmt(f), - TokenKind::String => write!(f, "string value"), + TokenKind::String => write!(f, "string"), TokenKind::Struct => Token::Struct.fmt(f), TokenKind::While => Token::While.fmt(f), }