jeff/dust
1
0
Fork 0
Code Issues

Clean up tokens

Browse Source
This commit is contained in:
Jeff 2024-08-09 01:55:34 -04:00
parent 83018ec5ec
commit 580b85e2d0
3 changed files with 74 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dust-lang/src/dust_error.rs
View File

@ -2,7 +2,7 @@
use annotate_snippets::{Level, Renderer, Snippet};
use std::{error::Error, fmt::Display};
use crate::{AnalyzerError, VmError};
use crate::VmError;
/// An error that occurred during the execution of the Dust language and its
/// corresponding source code.

9
dust-lang/src/lex.rs
View File

@ -182,10 +182,13 @@ impl Lexer {
/// Progress to the next character.
fn next_char(&mut self, source: &str) -> Option<char> {
source[self.position..].chars().next().map(|c| {
if let Some(c) = source[self.position..].chars().next() {
self.position += c.len_utf8();
c
})
Some(c)
} else {
None
}
}
/// Skip whitespace characters.

109
dust-lang/src/token.rs
View File

@ -62,43 +62,68 @@ impl<'src> Token<'src> {
Token::Minus => TokenOwned::Minus,
}
}
pub fn as_str(&self) -> &'static str {
match self {
Token::Eof => "EOF",
Token::Identifier(_) => "identifier",
Token::Boolean(_) => "boolean",
Token::Float(_) => "float",
Token::Integer(_) => "integer",
Token::String(_) => "string",
Token::IsEven => "is_even",
Token::IsOdd => "is_odd",
Token::Length => "length",
Token::ReadLine => "read_line",
Token::WriteLine => "write_line",
Token::Comma => ",",
Token::Dot => ".",
Token::Equal => "=",
Token::Plus => "+",
Token::Star => "*",
Token::LeftParenthesis => "(",
Token::RightParenthesis => ")",
Token::LeftSquareBrace => "[",
Token::RightSquareBrace => "]",
Token::Minus => "-",
}
}
}
impl<'src> Display for Token<'src> {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
Token::Eof => write!(f, "EOF"),
Token::Identifier(text) => write!(f, "{text}"),
Token::Boolean(boolean) => write!(f, "{boolean}"),
Token::Float(float) => write!(f, "{float}"),
Token::Integer(integer) => write!(f, "{integer}"),
Token::String(string) => write!(f, "{string}"),
Token::IsEven => write!(f, "is_even"),
Token::IsOdd => write!(f, "is_odd"),
Token::Length => write!(f, "length"),
Token::ReadLine => write!(f, "read_line"),
Token::WriteLine => write!(f, "write_line"),
Token::Comma => write!(f, ","),
Token::Dot => write!(f, "."),
Token::Equal => write!(f, "="),
Token::Plus => write!(f, "+"),
Token::Star => write!(f, "*"),
Token::LeftParenthesis => write!(f, "("),
Token::RightParenthesis => write!(f, ")"),
Token::LeftSquareBrace => write!(f, "["),
Token::RightSquareBrace => write!(f, "]"),
Token::Minus => write!(f, "-"),
}
f.write_str(self.as_str())
}
}
impl<'src> PartialEq for Token<'src> {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
// Floats are compared by their bit representation.
(Token::Float(left), Token::Float(right)) => left.to_bits() == right.to_bits(),
_ => {
matches!(self, other)
}
// Compare all other variants normally.
(Token::Eof, Token::Eof) => true,
(Token::Identifier(left), Token::Identifier(right)) => left == right,
(Token::Boolean(left), Token::Boolean(right)) => left == right,
(Token::Integer(left), Token::Integer(right)) => left == right,
(Token::String(left), Token::String(right)) => left == right,
(Token::IsEven, Token::IsEven) => true,
(Token::IsOdd, Token::IsOdd) => true,
(Token::Length, Token::Length) => true,
(Token::ReadLine, Token::ReadLine) => true,
(Token::WriteLine, Token::WriteLine) => true,
(Token::Comma, Token::Comma) => true,
(Token::Dot, Token::Dot) => true,
(Token::Equal, Token::Equal) => true,
(Token::Plus, Token::Plus) => true,
(Token::Star, Token::Star) => true,
(Token::LeftParenthesis, Token::LeftParenthesis) => true,
(Token::RightParenthesis, Token::RightParenthesis) => true,
(Token::LeftSquareBrace, Token::LeftSquareBrace) => true,
(Token::RightSquareBrace, Token::RightSquareBrace) => true,
(Token::Minus, Token::Minus) => true,
_ => false,
}
}
}
@ -141,27 +166,27 @@ pub enum TokenOwned {
impl Display for TokenOwned {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self {
TokenOwned::Eof => write!(f, "EOF"),
TokenOwned::Eof => Token::Eof.fmt(f),
TokenOwned::Identifier(text) => write!(f, "{text}"),
TokenOwned::Boolean(boolean) => write!(f, "{boolean}"),
TokenOwned::Float(float) => write!(f, "{float}"),
TokenOwned::Integer(integer) => write!(f, "{integer}"),
TokenOwned::String(string) => write!(f, "{string}"),
TokenOwned::IsEven => write!(f, "is_even"),
TokenOwned::IsOdd => write!(f, "is_odd"),
TokenOwned::Length => write!(f, "length"),
TokenOwned::ReadLine => write!(f, "read_line"),
TokenOwned::WriteLine => write!(f, "write_line"),
TokenOwned::Comma => write!(f, ","),
TokenOwned::Dot => write!(f, "."),
TokenOwned::Equal => write!(f, "="),
TokenOwned::Plus => write!(f, "+"),
TokenOwned::Star => write!(f, "*"),
TokenOwned::LeftParenthesis => write!(f, "("),
TokenOwned::RightParenthesis => write!(f, ")"),
TokenOwned::LeftSquareBrace => write!(f, "["),
TokenOwned::RightSquareBrace => write!(f, "]"),
TokenOwned::Minus => write!(f, "-"),
TokenOwned::IsEven => Token::IsEven.fmt(f),
TokenOwned::IsOdd => Token::IsOdd.fmt(f),
TokenOwned::Length => Token::Length.fmt(f),
TokenOwned::ReadLine => Token::ReadLine.fmt(f),
TokenOwned::WriteLine => Token::WriteLine.fmt(f),
TokenOwned::Comma => Token::Comma.fmt(f),
TokenOwned::Dot => Token::Dot.fmt(f),
TokenOwned::Equal => Token::Equal.fmt(f),
TokenOwned::Plus => Token::Plus.fmt(f),
TokenOwned::Star => Token::Star.fmt(f),
TokenOwned::LeftParenthesis => Token::LeftParenthesis.fmt(f),
TokenOwned::RightParenthesis => Token::RightParenthesis.fmt(f),
TokenOwned::LeftSquareBrace => Token::LeftSquareBrace.fmt(f),
TokenOwned::RightSquareBrace => Token::RightSquareBrace.fmt(f),
TokenOwned::Minus => Token::Minus.fmt(f),
}
}
}