From 580b85e2d0297d80e1d1d0a95fd04e93fca51905 Mon Sep 17 00:00:00 2001 From: Jeff Date: Fri, 9 Aug 2024 01:55:34 -0400 Subject: [PATCH] Clean up tokens --- dust-lang/src/dust_error.rs | 2 +- dust-lang/src/lex.rs | 9 ++- dust-lang/src/token.rs | 109 ++++++++++++++++++++++-------------- 3 files changed, 74 insertions(+), 46 deletions(-) diff --git a/dust-lang/src/dust_error.rs b/dust-lang/src/dust_error.rs index 1993b4f..ae6aa16 100644 --- a/dust-lang/src/dust_error.rs +++ b/dust-lang/src/dust_error.rs @@ -2,7 +2,7 @@ use annotate_snippets::{Level, Renderer, Snippet}; use std::{error::Error, fmt::Display}; -use crate::{AnalyzerError, VmError}; +use crate::VmError; /// An error that occurred during the execution of the Dust language and its /// corresponding source code. diff --git a/dust-lang/src/lex.rs b/dust-lang/src/lex.rs index 572a3e5..c543042 100644 --- a/dust-lang/src/lex.rs +++ b/dust-lang/src/lex.rs @@ -182,10 +182,13 @@ impl Lexer { /// Progress to the next character. fn next_char(&mut self, source: &str) -> Option { - source[self.position..].chars().next().map(|c| { + if let Some(c) = source[self.position..].chars().next() { self.position += c.len_utf8(); - c - }) + + Some(c) + } else { + None + } } /// Skip whitespace characters. diff --git a/dust-lang/src/token.rs b/dust-lang/src/token.rs index dfb6e83..db24fcd 100644 --- a/dust-lang/src/token.rs +++ b/dust-lang/src/token.rs @@ -62,43 +62,68 @@ impl<'src> Token<'src> { Token::Minus => TokenOwned::Minus, } } + + pub fn as_str(&self) -> &'static str { + match self { + Token::Eof => "EOF", + Token::Identifier(_) => "identifier", + Token::Boolean(_) => "boolean", + Token::Float(_) => "float", + Token::Integer(_) => "integer", + Token::String(_) => "string", + Token::IsEven => "is_even", + Token::IsOdd => "is_odd", + Token::Length => "length", + Token::ReadLine => "read_line", + Token::WriteLine => "write_line", + Token::Comma => ",", + Token::Dot => ".", + Token::Equal => "=", + Token::Plus => "+", + Token::Star => "*", + Token::LeftParenthesis => "(", + Token::RightParenthesis => ")", + Token::LeftSquareBrace => "[", + Token::RightSquareBrace => "]", + Token::Minus => "-", + } + } } impl<'src> Display for Token<'src> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - Token::Eof => write!(f, "EOF"), - Token::Identifier(text) => write!(f, "{text}"), - Token::Boolean(boolean) => write!(f, "{boolean}"), - Token::Float(float) => write!(f, "{float}"), - Token::Integer(integer) => write!(f, "{integer}"), - Token::String(string) => write!(f, "{string}"), - Token::IsEven => write!(f, "is_even"), - Token::IsOdd => write!(f, "is_odd"), - Token::Length => write!(f, "length"), - Token::ReadLine => write!(f, "read_line"), - Token::WriteLine => write!(f, "write_line"), - Token::Comma => write!(f, ","), - Token::Dot => write!(f, "."), - Token::Equal => write!(f, "="), - Token::Plus => write!(f, "+"), - Token::Star => write!(f, "*"), - Token::LeftParenthesis => write!(f, "("), - Token::RightParenthesis => write!(f, ")"), - Token::LeftSquareBrace => write!(f, "["), - Token::RightSquareBrace => write!(f, "]"), - Token::Minus => write!(f, "-"), - } + f.write_str(self.as_str()) } } impl<'src> PartialEq for Token<'src> { fn eq(&self, other: &Self) -> bool { match (self, other) { + // Floats are compared by their bit representation. (Token::Float(left), Token::Float(right)) => left.to_bits() == right.to_bits(), - _ => { - matches!(self, other) - } + + // Compare all other variants normally. + (Token::Eof, Token::Eof) => true, + (Token::Identifier(left), Token::Identifier(right)) => left == right, + (Token::Boolean(left), Token::Boolean(right)) => left == right, + (Token::Integer(left), Token::Integer(right)) => left == right, + (Token::String(left), Token::String(right)) => left == right, + (Token::IsEven, Token::IsEven) => true, + (Token::IsOdd, Token::IsOdd) => true, + (Token::Length, Token::Length) => true, + (Token::ReadLine, Token::ReadLine) => true, + (Token::WriteLine, Token::WriteLine) => true, + (Token::Comma, Token::Comma) => true, + (Token::Dot, Token::Dot) => true, + (Token::Equal, Token::Equal) => true, + (Token::Plus, Token::Plus) => true, + (Token::Star, Token::Star) => true, + (Token::LeftParenthesis, Token::LeftParenthesis) => true, + (Token::RightParenthesis, Token::RightParenthesis) => true, + (Token::LeftSquareBrace, Token::LeftSquareBrace) => true, + (Token::RightSquareBrace, Token::RightSquareBrace) => true, + (Token::Minus, Token::Minus) => true, + _ => false, } } } @@ -141,27 +166,27 @@ pub enum TokenOwned { impl Display for TokenOwned { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { - TokenOwned::Eof => write!(f, "EOF"), + TokenOwned::Eof => Token::Eof.fmt(f), TokenOwned::Identifier(text) => write!(f, "{text}"), TokenOwned::Boolean(boolean) => write!(f, "{boolean}"), TokenOwned::Float(float) => write!(f, "{float}"), TokenOwned::Integer(integer) => write!(f, "{integer}"), TokenOwned::String(string) => write!(f, "{string}"), - TokenOwned::IsEven => write!(f, "is_even"), - TokenOwned::IsOdd => write!(f, "is_odd"), - TokenOwned::Length => write!(f, "length"), - TokenOwned::ReadLine => write!(f, "read_line"), - TokenOwned::WriteLine => write!(f, "write_line"), - TokenOwned::Comma => write!(f, ","), - TokenOwned::Dot => write!(f, "."), - TokenOwned::Equal => write!(f, "="), - TokenOwned::Plus => write!(f, "+"), - TokenOwned::Star => write!(f, "*"), - TokenOwned::LeftParenthesis => write!(f, "("), - TokenOwned::RightParenthesis => write!(f, ")"), - TokenOwned::LeftSquareBrace => write!(f, "["), - TokenOwned::RightSquareBrace => write!(f, "]"), - TokenOwned::Minus => write!(f, "-"), + TokenOwned::IsEven => Token::IsEven.fmt(f), + TokenOwned::IsOdd => Token::IsOdd.fmt(f), + TokenOwned::Length => Token::Length.fmt(f), + TokenOwned::ReadLine => Token::ReadLine.fmt(f), + TokenOwned::WriteLine => Token::WriteLine.fmt(f), + TokenOwned::Comma => Token::Comma.fmt(f), + TokenOwned::Dot => Token::Dot.fmt(f), + TokenOwned::Equal => Token::Equal.fmt(f), + TokenOwned::Plus => Token::Plus.fmt(f), + TokenOwned::Star => Token::Star.fmt(f), + TokenOwned::LeftParenthesis => Token::LeftParenthesis.fmt(f), + TokenOwned::RightParenthesis => Token::RightParenthesis.fmt(f), + TokenOwned::LeftSquareBrace => Token::LeftSquareBrace.fmt(f), + TokenOwned::RightSquareBrace => Token::RightSquareBrace.fmt(f), + TokenOwned::Minus => Token::Minus.fmt(f), } } }